From 276d92fc857f5314e2f785272066e2a79276fb1a Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 00:42:45 +0800 Subject: [PATCH 01/32] Inner data structions change 1. **PcsFile** class - `path` Default is the name of the file. It will be different from different apis returned. See `AliPCSApi.meta`, `AliPCSApi.meta_by_path`, `AliPCSApi.get_file`, `AliPCSApi.list`, `AliPCSApi.list_iter`, `AliPCSApi.path_traceback`, `AliPCSApi.path`. - `update_download_url` The method is removed. Use `AliPCSApi.update_download_url` instead. 2. **FromTo** type The original `FromTo` is a nametuple. We change it to a general type `FromTo = Tuple[F, T]` 3. **PcsDownloadUrl** class - `expires` Add the method to check whether the `download_url` expires. --- alipcs_py/alipcs/inner.py | 99 ++++++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 42 deletions(-) diff --git a/alipcs_py/alipcs/inner.py b/alipcs_py/alipcs/inner.py index 371e1fc..93f23ce 100644 --- a/alipcs_py/alipcs/inner.py +++ b/alipcs_py/alipcs/inner.py @@ -1,17 +1,25 @@ -from typing import Optional, List, Dict, Any, TYPE_CHECKING +from typing import Optional, List, Dict, Any, TYPE_CHECKING, Tuple, TypeVar from dataclasses import dataclass -from collections import namedtuple import time import re import urllib.parse import warnings + from alipcs_py.common.date import iso_8601_to_timestamp, now_timestamp if TYPE_CHECKING: from alipcs_py.alipcs.api import AliPCSApi +# FromTo type is used to represent the direction of the file transfer +# When the direction is from local to remote, the type is FromTo[PathType, str] +# When the direction is from remote to local, the type is FromTo[str, PathType] +F = TypeVar("F") +T = TypeVar("T") +FromTo = Tuple[F, T] + + @dataclass class PcsRapidUploadInfo: """Rapid Upload Info""" @@ -51,21 +59,36 @@ def default_hash_link_protocol() -> str: @dataclass class PcsFile: - """ - A Ali PCS file - - path: str # remote absolute path - is_dir: Optional[bool] = None - is_file: Optional[bool] = None - fs_id: Optional[int] = None # file id - size: Optional[int] = None - md5: Optional[str] = None - block_list: Optional[List[str]] = None # block md5 list - category: Optional[int] = None - user_id: Optional[int] = None - created_at: Optional[int] = None # server created time - updated_at: Optional[int] = None # server updated time - shared: Optional[bool] = None # this file is shared if True + """The remote file/directory information + + Args: + file_id (str): The unique identifier of the file. + name (str): The name of the file. + parent_file_id (str): The unique identifier of the parent file. If the parent directory is '/', its id is 'root'. + type (str): The type of the file, such as "file" or "folder". + is_dir (bool): Indicates whether the file is a directory. + is_file (bool): Indicates whether the file is a regular file. + size (int): The size of the file in bytes. + path (str): The remote path of the file. Default is the name of the file. + created_at (Optional[int]): The timestamp of when the file was created on the server. + updated_at (Optional[int]): The timestamp of when the file was last updated on the server. + file_extension (Optional[str]): The file extension of the file. + content_type (Optional[str]): The content type of the file. + mime_type (Optional[str]): The MIME type of the file. + mime_extension (Optional[str]): The MIME type extension of the file. + labels (Optional[List[str]]): A list of labels associated with the file. + status (Optional[str]): The status of the file. + hidden (Optional[bool]): Indicates whether the file is hidden. + starred (Optional[bool]): Indicates whether the file is starred. + category (Optional[str]): The category of the file. + punish_flag (Optional[int]): A flag indicating whether the file has been punished. + encrypt_mode (Optional[str]): The encryption mode of the file. + drive_id (Optional[str]): The unique identifier of the drive the file is stored in. + domain_id (Optional[str]): The unique identifier of the domain the file is associated with. + upload_id (Optional[str]): The unique identifier of the file upload. + async_task_id (Optional[str]): The unique identifier of the asynchronous task associated with the file. + rapid_upload_info (Optional[PcsRapidUploadInfo]): Information about the rapid upload of the file. + download_url (Optional[str]): The URL for downloading the file. """ file_id: str @@ -75,10 +98,10 @@ class PcsFile: is_dir: bool is_file: bool size: int = 0 - path: str = "" # remote absolute path + path: str = "" created_at: Optional[int] = None # server created time - updated_at: Optional[int] = None # server updated time (updated time) + updated_at: Optional[int] = None # server updated time file_extension: Optional[str] = None content_type: Optional[str] = None @@ -120,14 +143,16 @@ def from_(info) -> "PcsFile": name=info.get("name"), ) + filename = info.get("name") or info.get("file_name", "") return PcsFile( file_id=info.get("file_id", ""), - name=info.get("name", ""), + name=filename, parent_file_id=info.get("parent_file_id", ""), type=info.get("type", ""), is_dir=info.get("type") == "folder", is_file=info.get("type") == "file", size=info.get("size"), + path=filename, # Default path is the name of the file created_at=created_at, updated_at=updated_at, file_extension=info.get("file_extension"), @@ -176,25 +201,6 @@ def download_url_expires(self) -> bool: return False return True - def update_download_url(self, api: "AliPCSApi"): - """Update the download url if it expires""" - - warnings.warn( - "This method is deprecated and will be removed in a future version, use `update_download_url` in `AliPCSApi` instead", - DeprecationWarning, - ) - - if self.is_file: - if self.download_url_expires(): - pcs_url = api.download_link(self.file_id) - if pcs_url: - self.download_url = pcs_url.url - else: - if getattr(api, "_aliopenpcsapi", None): - pcs_url = api.download_link(self.file_id) - if pcs_url: - self.download_url = pcs_url.url - @dataclass class PcsUploadUrl: @@ -419,9 +425,6 @@ def is_expired(self) -> bool: return time.time() >= self.expire_time -FromTo = namedtuple("FromTo", ["from_", "to_"]) - - @dataclass class PcsSpace: used_size: int @@ -593,6 +596,18 @@ class PcsDownloadUrl: expiration: Optional[int] = None ratelimit: Optional[PcsRateLimit] = None + def expires(self) -> bool: + """Check whether the `self.download_url` expires""" + + url = self.download_url or self.url + if url: + mod = re.search(r"oss-expires=(\d+)", url) + if mod: + expire_time = float(mod.group(1)) + if time.time() < expire_time - 5: + return False + return True + @staticmethod def from_(info) -> "PcsDownloadUrl": expiration = None From 03a7612c7dd309a35c2a0cb1c23d8a1ace48067e Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 10:53:52 +0800 Subject: [PATCH 02/32] Unneeded `walk` function using `os.walk` instead --- alipcs_py/common/path.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/alipcs_py/common/path.py b/alipcs_py/common/path.py index bdda953..0aeeff2 100644 --- a/alipcs_py/common/path.py +++ b/alipcs_py/common/path.py @@ -1,11 +1,10 @@ -from typing import Tuple, Union, Iterator +from typing import Tuple, Union from pathlib import Path, PurePosixPath -import os from os import PathLike from alipcs_py.common.platform import IS_WIN -PathType = Union["str", PathLike, Path] +PathType = Union[str, PathLike, Path] def exists(localpath: PathType) -> bool: @@ -23,13 +22,7 @@ def is_dir(localpath: PathType) -> bool: return localpath.is_dir() -def walk(localpath: PathType) -> Iterator[str]: - for root, _, files in os.walk(Path(localpath)): - r = Path(root) - for fl in files: - yield (r / fl).as_posix() - - +# TODO: Change function name to `join_path_as_posix` def join_path(parent: PathType, *children: PathType) -> str: """Join posix paths""" From 2a3882120ebf1d921bad189182cad7eb99555cc2 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 11:17:52 +0800 Subject: [PATCH 03/32] Add `make_http_session` function to generate http session with connection pool parameters --- alipcs_py/common/net.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/alipcs_py/common/net.py b/alipcs_py/common/net.py index b4d4d91..66ac2a7 100644 --- a/alipcs_py/common/net.py +++ b/alipcs_py/common/net.py @@ -1,5 +1,8 @@ import socket +import requests +import requests.adapters + def avail_port(port: int) -> bool: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: @@ -20,3 +23,22 @@ def random_avail_port() -> int: s.listen() _, port = s.getsockname() return port + + +def make_http_session( + max_keepalive_connections: int = 50, + max_connections: int = 50, + keepalive_expiry: float = 10 * 60, + max_retries: int = 2, +) -> requests.Session: + """Make a http session with keepalive connections, maximum connections and retries""" + + session = requests.Session() + adapter = requests.adapters.HTTPAdapter( + pool_connections=max_keepalive_connections, + pool_maxsize=max_connections, + max_retries=max_retries, + ) + session.mount("http://", adapter) + session.mount("https://", adapter) + return session From 821d20d94b1636ceb8a8070d7b774222b3b7d1e2 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 11:19:50 +0800 Subject: [PATCH 04/32] Ignore existed directories when creating them --- alipcs_py/common/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alipcs_py/common/log.py b/alipcs_py/common/log.py index 4b2a09b..4e2e95e 100644 --- a/alipcs_py/common/log.py +++ b/alipcs_py/common/log.py @@ -31,7 +31,7 @@ def get_logger( filename = Path(filename) _dir = filename.parent if not _dir.exists(): - _dir.mkdir() + _dir.mkdir(parents=True, exist_ok=True) file_handler = logging.FileHandler(filename) file_handler.setFormatter(logging.Formatter(fmt)) From f6b9648dd32fd4370a150101e1a7e2c0e3330eb7 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 11:20:46 +0800 Subject: [PATCH 05/32] Set RangeRequestIO to be readable --- alipcs_py/common/io.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/alipcs_py/common/io.py b/alipcs_py/common/io.py index 12dcdd4..e651544 100644 --- a/alipcs_py/common/io.py +++ b/alipcs_py/common/io.py @@ -1017,7 +1017,7 @@ def __init__( url: str, headers: Optional[Dict[str, str]] = None, max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE, - callback: Callable[..., None] = None, + callback: Optional[Callable[..., None]] = None, encrypt_password: bytes = b"", **kwargs, ): @@ -1096,6 +1096,9 @@ def seek(self, offset: int, whence: int = 0) -> int: def tell(self) -> int: return self._offset + def readable(self) -> bool: + return True + def seekable(self) -> bool: return self._auto_decrypt_request.rangeable() From beea3e5aa10dd176afe2660299517ce21bbb8db9 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 11:23:28 +0800 Subject: [PATCH 06/32] `Executor` is equal to `ThreadPoolExecutor`, so remove it --- alipcs_py/common/concurrent.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/alipcs_py/common/concurrent.py b/alipcs_py/common/concurrent.py index 31649f6..0cc14ae 100644 --- a/alipcs_py/common/concurrent.py +++ b/alipcs_py/common/concurrent.py @@ -1,4 +1,3 @@ -from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Optional, Callable, Any from functools import wraps from threading import Semaphore @@ -37,33 +36,3 @@ def retry_it(*args, **kwargs): return retry_it return wrap - - -class Executor: - """ - Executor is a ThreadPoolExecutor when max_workers > 1, else a single thread executor. - """ - - def __init__(self, max_workers: int = 1): - self._max_workers = max_workers - self._pool = ThreadPoolExecutor(max_workers=max_workers) if max_workers > 1 else None - self._semaphore = Semaphore(max_workers) - self._futures = [] - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if self._pool is not None: - as_completed(self._futures) - self._pool.shutdown() - self._futures.clear() - - def submit(self, func, *args, **kwargs): - if self._pool is not None: - self._semaphore.acquire() - fut = self._pool.submit(sure_release, self._semaphore, func, *args, **kwargs) - self._futures.append(fut) - return fut - else: - return func(*args, **kwargs) From f7115a111c4f7036f4b79f6a710a8ef5d6a38a82 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 11:26:08 +0800 Subject: [PATCH 07/32] Argument `retries` is renamed to `max_retries` --- alipcs_py/common/downloader.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/alipcs_py/common/downloader.py b/alipcs_py/common/downloader.py index 3c0ae1d..f06f02c 100644 --- a/alipcs_py/common/downloader.py +++ b/alipcs_py/common/downloader.py @@ -1,9 +1,9 @@ from typing import Optional, Any, Callable -from os import PathLike from pathlib import Path from alipcs_py.common.io import RangeRequestIO from alipcs_py.common.concurrent import retry +from alipcs_py.common.path import PathType DEFAULT_MAX_WORKERS = 5 @@ -13,16 +13,16 @@ class MeDownloader: def __init__( self, range_request_io: RangeRequestIO, - localpath: PathLike, + localpath: PathType, continue_: bool = False, - retries: int = 2, + max_retries: int = 2, done_callback: Optional[Callable[..., Any]] = None, except_callback: Optional[Callable[[Exception], Any]] = None, ) -> None: self.range_request_io = range_request_io self.localpath = localpath self.continue_ = continue_ - self.retries = retries + self.max_retries = max_retries self.done_callback = done_callback self.except_callback = except_callback @@ -53,7 +53,7 @@ def download(self): """ @retry( - self.retries, + self.max_retries, except_callback=lambda err, fails: ( self.range_request_io.reset(), self.except_callback(err) if self.except_callback else None, From 8349985c1fc36dd05cf1c8f629c6399118ee9688 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 12:14:42 +0800 Subject: [PATCH 08/32] Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. **AliPCSBaseError** class The base Exception class used for the PCS errors. 2. **AliPCSError(AliPCSBaseError)** class The error returned from alipan server when the client’s request is incorrect or the token is expired. It throw at **AliPCS** class when an error occurs. 3. **DownloadError(AliPCSBaseError)** class An error occurs when downloading action fails. 4. **UploadError(AliPCSBaseError)** class An error occurs when uploading action fails. 5. **RapidUploadError(UploadError)** class An error occurred when rapid uploading action fails. 6. **make_alipcs_error** function Make an AliPCSError instance. --- alipcs_py/alipcs/errors.py | 65 ++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/alipcs_py/alipcs/errors.py b/alipcs_py/alipcs/errors.py index 8213e46..f6ba471 100644 --- a/alipcs_py/alipcs/errors.py +++ b/alipcs_py/alipcs/errors.py @@ -3,18 +3,65 @@ import time import logging +from alipcs_py.common.path import PathType +from alipcs_py.alipcs.inner import PcsFile + logger = logging.getLogger(__name__) -class AliPCSError(Exception): - def __init__(self, message: str, error_code: Optional[str] = None, cause=None): - self.__cause__ = cause +class AliPCSBaseError(Exception): + """Base exception for all errors. + + Args: + message (Optional[object]): The message object stringified as 'message' attribute + keyword error (Exception): The original exception if any + """ + + def __init__(self, message: Optional[object], *args: Any, **kwargs: Any) -> None: + self.inner_exception: Optional[BaseException] = kwargs.get("error") + + self.message = str(message) + super().__init__(self.message, *args) + + +class AliPCSError(AliPCSBaseError): + """The error returned from alipan server when the client’s request is incorrect or the token is expired. + + It is throwed at `AliPCS` class when an error occurs, then transports to the upper level class. + """ + + def __init__(self, message: str, error_code: Optional[str] = None): self.error_code = error_code super().__init__(message) -def parse_error(error_code: str, info: Any = None) -> AliPCSError: - msg = f"error_code: {error_code}, response: {info}" +class DownloadError(AliPCSBaseError): + """An error occurred while downloading a file.""" + + def __init__(self, message: str, remote_pcs_file: PcsFile, localdir: PathType): + self.remote_pcs_file = remote_pcs_file + self.localdir = localdir + super().__init__(message) + + +class UploadError(AliPCSBaseError): + """An error occurred while uploading a file.""" + + def __init__(self, message: str, localpath: PathType, remotepath: str): + self.local_file = localpath + self.remote_dir = remotepath + super().__init__(message) + + +class RapidUploadError(UploadError): + """An error occurred while rapid uploading a file.""" + + def __init__(self, message: str, localpath: PathType, remotepath: str): + super().__init__(message, localpath, remotepath) + + +def make_alipcs_error(error_code: str, info: Any = None) -> AliPCSError: + msg = f"API error code: {error_code}, response: {info}" return AliPCSError(msg, error_code=error_code) @@ -27,7 +74,7 @@ def check(*args, **kwargs): error_code = info.get("code") if error_code: - err = parse_error(error_code, str(info)) + err = make_alipcs_error(error_code, str(info)) raise err return info @@ -70,8 +117,12 @@ def refresh(*args, **kwargs): self._signature = "" continue + elif code == "NotFound.File" and func.__name__ == "meta": + # keep the not found file info for meta api + return info + return info - raise parse_error(code) + raise make_alipcs_error(code) return refresh From b079100c258e3e798d16e020a91ae930817c3d67 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 17:50:02 +0800 Subject: [PATCH 09/32] **AliPCS** class APIs New/Update ```python class AliPCS: SHARE_AUTHS: Dict[str, SharedAuth] = {} def __init__( self, refresh_token: str, access_token: str = "", token_type: str = "Bearer", expire_time: int = 0, user_id: str = "", user_name: str = "", nick_name: str = "", device_id: str = "", default_drive_id: str = "", role: str = "", status: str = "", error_max_retries: int = 2, max_keepalive_connections: int = 50, max_connections: int = 50, keepalive_expiry: float = 10 * 60, connection_max_retries: int = 2, ): ... ``` The core alipan.com service apis. It directly handles the raw requests and responses of the service. **New/Changed APIs are following:** - `path_traceback` method Traceback the path of the file by its file_id. Return the list of all parent directories' info from the file to the top level directory. - `meta_by_path` method Get meta info of the file by its path. > Can not get the shared files' meta info. - `meta` method Get meta info of the file by its file_id. - `exists` method Check whether the file exists. Return True if the file exists and does not in the trash else False. - `exists_in_trash` method Check whether the file exists in the trash. Return True if the file exists in the trash else False. - `walk` method Walk through the directory tree by its file_id. - `download_link` method Get download link of the file by its file_id. First try to get the download link from the meta info of the file. If the download link is not in the meta info, then request the getting download url api. --- alipcs_py/alipcs/pcs.py | 394 ++++++++++++++++++++++++++++++---------- 1 file changed, 298 insertions(+), 96 deletions(-) diff --git a/alipcs_py/alipcs/pcs.py b/alipcs_py/alipcs/pcs.py index 6cc5ae5..5023175 100644 --- a/alipcs_py/alipcs/pcs.py +++ b/alipcs_py/alipcs/pcs.py @@ -12,9 +12,9 @@ from alipcs_py.common.io import RangeRequestIO, DEFAULT_MAX_CHUNK_SIZE, ChunkIO, total_len from alipcs_py.common.cache import timeout_cache from alipcs_py.common.crypto import generate_secp256k1_keys -from alipcs_py.alipcs.errors import AliPCSError, parse_error, handle_error -from alipcs_py.alipcs.errors import assert_ok +from alipcs_py.alipcs.errors import AliPCSError, make_alipcs_error, handle_error from alipcs_py.alipcs.inner import SharedAuth +from alipcs_py.common.net import make_http_session UPLOAD_CHUNK_SIZE = 10 * constant.OneM @@ -25,8 +25,7 @@ ALIYUNDRIVE_COM_API = "https://api.aliyundrive.com" ALIYUNDRIVE_OPENAPI_DOMAIN = "https://openapi.aliyundrive.com" -# TODO: Update UA -PCS_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36" +PCS_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36" PCS_HEADERS = {"Origin": ALIYUNDRIVE_COM, "Referer": ALIYUNDRIVE_COM + "/", "User-Agent": PCS_UA} CheckNameMode = Literal[ @@ -51,6 +50,9 @@ class PcsNode(Enum): CreateSession = "users/v1/users/device/create_session" FileList = "adrive/v3/file/list" + Walk = "v2/file/walk" + GetPath = "adrive/v1/file/get_path" + MetaByPath = "v2/file/get_by_path" Meta = "v2/file/get" Search = "adrive/v3/file/search" DownloadUrl = "v2/file/get_download_url" @@ -86,7 +88,31 @@ def url(self) -> str: class AliPCS: - """`AliPCS` provides pcs's apis which return raw json""" + """Aliyun Drive Personal Cloud Service Raw API + + The core class is used to interact with Aliyun Drive Personal Cloud Service. + It provides the basic operations of the service and handles the raw requests and responses. + + An `AliPCSError` error will be raised if the code of the response occurs. + + Args: + refresh_token (str): The refresh token of the user. + access_token (str, optional): The access token of the user. + token_type (str): The token type. Default is "Bearer". + expire_time (int, optional): The expire time of the token. + user_id (str, optional): The user id of the user. + user_name (str, optional): The user name of the user. + nick_name (str, optional): The nick name of the user. + device_id (str, optional): The device id of the user. + default_drive_id (str, optional): The default drive id of the user. + role (str, optional): The role of the user. + status (str, optional): The status of the user. + error_max_retries (int): The max retries when a client request error occurs. Default is 2. + max_keepalive_connections (int): The max keepalive connections. Default is 50. + max_connections (int): The max number of connections in the pool. Default is 50. + keepalive_expiry (float): The keepalive expiry. Default is 10 * 60 seconds. + connection_max_retries (int): The max retries when a connection error occurs. Default is 2. + """ SHARE_AUTHS: Dict[str, SharedAuth] = {} @@ -103,8 +129,21 @@ def __init__( default_drive_id: str = "", role: str = "", status: str = "", + error_max_retries: int = 2, + max_keepalive_connections: int = 50, + max_connections: int = 50, + keepalive_expiry: float = 10 * 60, + connection_max_retries: int = 2, ): - self._session = requests.Session() + self._session = make_http_session( + max_keepalive_connections=max_keepalive_connections, + max_connections=max_connections, + keepalive_expiry=keepalive_expiry, + max_retries=connection_max_retries, + ) + + self._error_max_retries = error_max_retries + self._refresh_token = refresh_token self._access_token = access_token self._token_type = token_type @@ -244,7 +283,7 @@ def _request( ) return resp except Exception as err: - raise AliPCSError("AliPCS._request", cause=err) + raise AliPCSError("AliPCS._request") from err def refresh(self): """Refresh token""" @@ -255,7 +294,7 @@ def refresh(self): info = resp.json() if "code" in info: - err = parse_error(info["code"], info=info) + err = make_alipcs_error(info["code"], info=info) raise err self._user_id = info["user_id"] @@ -307,8 +346,14 @@ def do(self): return info - def meta(self, *file_ids: str, share_id: str = None): - assert "root" not in file_ids, '"root" has NOT meta info' + @handle_error + def path_traceback(self, file_id: str, share_id: Optional[str] = None): + """Traceback the path of the file by its file_id + + Return the list of all parent directories' info from the file to the top level directory. + """ + + url = PcsNode.GetPath.url() headers = dict(PCS_HEADERS) headers.update({"x-device-id": self.device_id, "x-signature": self.signature}) @@ -319,47 +364,128 @@ def meta(self, *file_ids: str, share_id: str = None): headers["x-share-token"] = share_token - responses = [] - for file_id in file_ids: - data = dict( - file_id=file_id, - fields="*", - image_thumbnail_process="image/resize,w_400/format,jpeg", - image_url_process="image/resize,w_375/format,jpeg", - video_thumbnail_process="video/snapshot,t_1000,f_jpg,ar_auto,w_375", - ) + data = dict(file_id=file_id, fields="*") - if share_id: - data["share_id"] = share_id - else: - data["drive_id"] = self.default_drive_id + if share_id: + data["share_id"] = share_id + else: + data["drive_id"] = self.default_drive_id - url = PcsNode.Meta.url() - resp = self._request(Method.Post, url, headers=headers, json=data) - info = resp.json() - responses.append(dict(body=info)) + resp = self._request(Method.Post, url, headers=headers, json=data) + info = resp.json() + return info + + @handle_error + def meta_by_path(self, remotepath: str): + """Get meta info of the file by its path - return dict(responses=responses) + Can not get the shared files' meta info. + """ + + assert remotepath.startswith("/"), "Path should start with '/'" + + url = PcsNode.MetaByPath.url() + headers = dict(PCS_HEADERS) + headers.update({"x-device-id": self.device_id, "x-signature": self.signature}) + + data = dict( + file_path=remotepath, + fields="*", + drive_id=self.default_drive_id, + image_thumbnail_process="image/resize,w_400/format,jpeg", + image_url_process="image/resize,w_375/format,jpeg", + video_thumbnail_process="video/snapshot,t_1000,f_jpg,ar_auto,w_375", + ) + + resp = self._request(Method.Post, url, headers=headers, json=data) + info = resp.json() + return info + + @handle_error + def meta(self, file_id: str, share_id: Optional[str] = None): + """Get meta info of the file by its file_id""" + + url = PcsNode.Meta.url() + headers = dict(PCS_HEADERS) + headers.update({"x-device-id": self.device_id, "x-signature": self.signature}) + + if share_id: + share_token = self.share_token(share_id) + assert share_token, "Need share_token" + + headers["x-share-token"] = share_token + + data = dict( + file_id=file_id, + fields="*", + image_thumbnail_process="image/resize,w_400/format,jpeg", + image_url_process="image/resize,w_375/format,jpeg", + video_thumbnail_process="video/snapshot,t_1000,f_jpg,ar_auto,w_375", + ) + + if share_id: + data["share_id"] = share_id + else: + data["drive_id"] = self.default_drive_id + + resp = self._request(Method.Post, url, headers=headers, json=data) + info = resp.json() + return info def exists(self, file_id: str) -> bool: + """Check whether the file exists + + Return True if the file exists and does not in the trash else False. + """ + if file_id == "root": return True - r = self.meta(file_id) - info = r["responses"][0]["body"] - if info.get("code") == "NotFound.File": + try: + info = self.meta(file_id) + except AliPCSError as err: + if err.error_code and err.error_code.startswith("NotFound."): + return False + raise err + + if info.get("parent_file_id") == "recyclebin": + # The file is not exists or in the trash return False else: return True + def exists_in_trash(self, file_id: str) -> bool: + """Check whether the file exists in the trash + + Return True if the file exists in the trash else False. + """ + + if file_id == "recyclebin": + return True + + try: + info = self.meta(file_id) + except AliPCSError as err: + if err.error_code and err.error_code.startswith("NotFound."): + return False + raise err + + if info.get("parent_file_id") == "recyclebin": + return True + else: + return False + def is_file(self, file_id: str) -> bool: if file_id == "root": return False - r = self.meta(file_id) - info = r["responses"][0]["body"] - if info.get("code") == "NotFound.File": - return False + try: + info = self.meta(file_id) + except AliPCSError as err: + if err.error_code and err.error_code.startswith("NotFound."): + return False + raise err + if info["type"] == "file": return True else: @@ -369,21 +495,23 @@ def is_dir(self, file_id: str) -> bool: if file_id == "root": return True - r = self.meta(file_id) - info = r["responses"][0]["body"] - if info.get("code") == "NotFound.File": - return False + try: + info = self.meta(file_id) + except AliPCSError as err: + if err.error_code and err.error_code.startswith("NotFound."): + return False + raise err + if info["type"] == "folder": return True else: return False - @assert_ok @handle_error def list( self, file_id: str, - share_id: str = "", + share_id: Optional[str] = None, desc: bool = False, name: bool = False, time: bool = False, @@ -441,6 +569,49 @@ def list( resp = self._request(Method.Post, url, headers=headers, json=data) return resp.json() + @handle_error + def walk( + self, + file_id: str, + share_id: Optional[str] = None, + all: bool = False, + limit: int = 200, + url_expire_sec: int = 14400, + next_marker: str = "", + ): + """Walk through the directory tree which has `file_id`""" + + url = PcsNode.Walk.url() + + headers = dict(PCS_HEADERS) + headers.update({"x-device-id": self.device_id, "x-signature": self.signature}) + + data = dict( + all=all, + drive_id=self.default_drive_id, + fields="*", + limit=limit, + parent_file_id=file_id, + url_expire_sec=url_expire_sec, + image_thumbnail_process="image/resize,w_256/format,jpeg", + image_url_process="image/resize,w_1920/format,jpeg/interlace,1", + video_thumbnail_process="video/snapshot,t_1000,f_jpg,ar_auto,w_256", + ) + if next_marker: + data["marker"] = next_marker + + if share_id: + share_token = self.share_token(share_id) + assert share_token, "Need share_token" + + data["share_id"] = share_id + data.pop("drive_id") + headers["x-share-token"] = share_token + + resp = self._request(Method.Post, url, headers=headers, json=data) + info = resp.json() + return info + @staticmethod def part_info_list(part_number: int) -> List[Dict[str, int]]: return [dict(part_number=i) for i in range(1, part_number + 1)] @@ -502,6 +673,7 @@ def create_file( def prepare_file(self, filename: str, dir_id: str, size: int, pre_hash: str = "", part_number: int = 1): return self.create_file(filename, dir_id, size, pre_hash=pre_hash, part_number=part_number) + @handle_error def get_upload_url(self, upload_id: str, file_id: str, part_number: int): """Get upload slices' urls @@ -520,8 +692,6 @@ def get_upload_url(self, upload_id: str, file_id: str, part_number: int): resp = self._request(Method.Post, url, json=data) return resp.json() - @assert_ok - @handle_error def rapid_upload_file(self, filename: str, dir_id: str, size: int, content_hash: str, proof_code: str): """Rapid Upload File @@ -531,7 +701,9 @@ def rapid_upload_file(self, filename: str, dir_id: str, size: int, content_hash: return self.create_file(filename, dir_id, size, content_hash=content_hash, proof_code=proof_code) - def upload_slice(self, io: IO, url: str, callback: Callable[[MultipartEncoderMonitor], None] = None) -> None: + def upload_slice( + self, io: IO, url: str, callback: Optional[Callable[[MultipartEncoderMonitor], None]] = None + ) -> None: """Upload the content of io to remote url""" cio = ChunkIO(io, total_len(io)) @@ -546,7 +718,6 @@ def upload_slice(self, io: IO, url: str, callback: Callable[[MultipartEncoderMon timeout=(3, 9), # (connect timeout, read timeout) ) - @assert_ok @handle_error def upload_complete(self, file_id: str, upload_id: str): url = PcsNode.UploadComplete.url() @@ -554,7 +725,6 @@ def upload_complete(self, file_id: str, upload_id: str): resp = self._request(Method.Post, url, json=data) return resp.json() - @assert_ok @handle_error def search( self, @@ -592,7 +762,6 @@ def search( resp = self._request(Method.Post, url, json=data) return resp.json() - @assert_ok @handle_error def makedir(self, dir_id: str, name: str): url = PcsNode.CreateWithFolders.url() @@ -606,7 +775,6 @@ def makedir(self, dir_id: str, name: str): resp = self._request(Method.Post, url, json=data) return resp.json() - @assert_ok @handle_error def batch_operate( self, requests_: List[Dict[str, Any]], resource: str = "file", headers: Optional[Dict[str, str]] = None @@ -648,7 +816,6 @@ def move(self, *file_ids: str): return self.batch_operate(requests_, resource="file") - @assert_ok @handle_error def rename(self, file_id: str, name: str): """Rename the file to `name`""" @@ -698,7 +865,6 @@ def remove(self, *file_ids: str): requests_.append(req) return self.batch_operate(requests_, resource="file") - @assert_ok @handle_error def check_available(self, file_ids: str): """Check whether file_ids are available""" @@ -711,7 +877,6 @@ def check_available(self, file_ids: str): resp = self._request(Method.Post, url, json=data) return resp.json() - @assert_ok @handle_error def share(self, *file_ids: str, password: str = "", period: int = 0, description: str = ""): """Share `remotepaths` to public @@ -734,7 +899,6 @@ def share(self, *file_ids: str, password: str = "", period: int = 0, description resp = self._request(Method.Post, url, json=data) return resp.json() - @assert_ok @handle_error def list_shared(self, next_marker: str = ""): """List shared links""" @@ -764,7 +928,6 @@ def cancel_shared(self, *share_ids: str): return self.batch_operate(requests_, resource="file") - @assert_ok @handle_error def get_share_token(self, share_id: str, share_password: str = ""): """Get share token""" @@ -792,7 +955,6 @@ def share_token(self, share_id: str) -> str: shared_auth = self.__class__.SHARE_AUTHS[share_id] return shared_auth.share_token - @assert_ok @handle_error def shared_info(self, share_id: str): """Get shared items info""" @@ -838,7 +1000,6 @@ def transfer_shared_files( return self.batch_operate(requests_, resource="file", headers=headers) - @assert_ok @handle_error def _get_shared_file_download_url(self, shared_file_id: str, share_id: str, expire_duration: int = 10 * 60): url = PcsNode.SharedFileDownloadUrl.url() @@ -862,7 +1023,6 @@ def shared_file_download_url(self, shared_file_id: str, share_id: str, expire_du resp = requests.get(url, headers=headers, allow_redirects=False) return resp.headers["Location"] - @assert_ok @handle_error def user_info(self): url = PcsNode.User.url() @@ -882,10 +1042,15 @@ def user_info(self): return {**info1, **info2, "user_vip_info": info3} @timeout_cache(1 * 60 * 60) # 1 hour timeout - @assert_ok @handle_error def download_link(self, file_id: str): - info = self.meta(file_id)["responses"][0]["body"] + """Get download link of the file by its file_id + + First try to get the download link from the meta info of the file. + If the download link is not in the meta info, then request the getting download url api. + """ + + info = self.meta(file_id) if info.get("url") or info.get("download_url"): return info @@ -900,7 +1065,7 @@ def file_stream( self, file_id: str, max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE, - callback: Callable[..., None] = None, + callback: Optional[Callable[..., None]] = None, encrypt_password: bytes = b"", ) -> Optional[RangeRequestIO]: info = self.download_link(file_id) @@ -928,7 +1093,7 @@ def shared_file_stream( share_id: str, expire_duration: int = 10 * 60, max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE, - callback: Callable[..., None] = None, + callback: Optional[Callable[..., None]] = None, encrypt_password: bytes = b"", ) -> Optional[RangeRequestIO]: url = self.shared_file_download_url(shared_file_id, share_id, expire_duration=expire_duration) @@ -995,7 +1160,31 @@ def url(self) -> str: class AliOpenPCS: - """`Aliyundrive Open PCS` provides pcs's apis which return raw json""" + """Aliyun Drive Personal Cloud Service Raw Open API + + The core class is used to interact with Aliyun Drive Personal Cloud Service with open apis. + It provides the basic operations of the service and handles the raw requests and responses. + + An `AliPCSError` error will be raised if the code of the response occurs. + + Args: + refresh_token (str): The refresh token of the user. + access_token (str, optional): The access token of the user. + token_type (str): The token type. Default is "Bearer". + expire_time (int, optional): The expire time of the token. + user_id (str, optional): The user id of the user. + user_name (str, optional): The user name of the user. + nick_name (str, optional): The nick name of the user. + device_id (str, optional): The device id of the user. + default_drive_id (str, optional): The default drive id of the user. + role (str, optional): The role of the user. + status (str, optional): The status of the user. + error_max_retries (int): The max retries when a client request error occurs. Default is 2. + max_keepalive_connections (int): The max keepalive connections. Default is 50. + max_connections (int): The max number of connections in the pool. Default is 50. + keepalive_expiry (float): The keepalive expiry. Default is 10 * 60 seconds. + connection_max_retries (int): The max retries when a connection error occurs. Default is 2. + """ SHARE_AUTHS: Dict[str, SharedAuth] = {} @@ -1014,12 +1203,24 @@ def __init__( default_drive_id: str = "", role: str = "", status: str = "", + error_max_retries: int = 2, + max_keepalive_connections: int = 50, + max_connections: int = 50, + keepalive_expiry: float = 10 * 60, + connection_max_retries: int = 2, ): assert ( client_id and client_secret ) or client_server, "(client_id and client_secret) or client_server must be set" - self._session = requests.Session() + self._session = make_http_session( + max_keepalive_connections=max_keepalive_connections, + max_connections=max_connections, + keepalive_expiry=keepalive_expiry, + max_retries=connection_max_retries, + ) + + self._error_max_retries = error_max_retries self._refresh_token = refresh_token self._access_token = access_token @@ -1158,7 +1359,7 @@ def _request( ) return resp except Exception as err: - raise AliPCSError("AliPCS._request", cause=err) + raise AliPCSError("AliPCS._request") from err def _update_refresh_token(self): """Update refresh token""" @@ -1175,7 +1376,7 @@ def _update_refresh_token(self): info = resp.json() if "code" in info: - err = parse_error(info["code"], info=info) + err = make_alipcs_error(info["code"], info=info) raise err self._refresh_token = info["refresh_token"] @@ -1195,7 +1396,7 @@ def _get_drive_info(self): info = resp.json() if "code" in info: - err = parse_error(info["code"], info=info) + err = make_alipcs_error(info["code"], info=info) raise err self._user_id = info["user_id"] @@ -1208,38 +1409,38 @@ def _get_drive_info(self): return info - def meta(self, *file_ids: str, share_id: str = None): - assert "root" not in file_ids, '"root" has NOT meta info' - - responses = [] - for file_id in file_ids: - data = dict(file_id=file_id, fields="*", drive_id=self.default_drive_id) - url = OpenPcsNode.Meta.url() - resp = self._request(Method.Post, url, json=data) - info = resp.json() - responses.append(dict(body=info)) - - return dict(responses=responses) + @handle_error + def meta(self, file_id: str, share_id: Optional[str] = None): + data = dict(file_id=file_id, fields="*", drive_id=self.default_drive_id) + url = OpenPcsNode.Meta.url() + resp = self._request(Method.Post, url, json=data) + info = resp.json() + return info def exists(self, file_id: str) -> bool: if file_id == "root": return True - r = self.meta(file_id) - info = r["responses"][0]["body"] - if info.get("code") == "NotFound.File": - return False - else: - return True + try: + info = self.meta(file_id) + except AliPCSError as err: + if err.error_code and err.error_code.startswith("NotFound."): + return False + raise err + + return True def is_file(self, file_id: str) -> bool: if file_id == "root": return False - r = self.meta(file_id) - info = r["responses"][0]["body"] - if info.get("code") == "NotFound.File": - return False + try: + info = self.meta(file_id) + except AliPCSError as err: + if err.error_code and err.error_code.startswith("NotFound."): + return False + raise err + if info["type"] == "file": return True else: @@ -1249,21 +1450,23 @@ def is_dir(self, file_id: str) -> bool: if file_id == "root": return True - r = self.meta(file_id) - info = r["responses"][0]["body"] - if info.get("code") == "NotFound.File": - return False + try: + info = self.meta(file_id) + except AliPCSError as err: + if err.error_code and err.error_code.startswith("NotFound."): + return False + raise err + if info["type"] == "folder": return True else: return False - @assert_ok @handle_error def list( self, file_id: str, - share_id: str = None, + share_id: Optional[str] = None, desc: bool = False, name: bool = False, time: bool = False, @@ -1312,7 +1515,6 @@ def part_info_list(part_number: int) -> List[Dict[str, int]]: return [dict(part_number=i) for i in range(1, part_number + 1)] @timeout_cache(4 * 60 * 60) # 4 hour timeout - @assert_ok @handle_error def download_link(self, file_id: str): url = OpenPcsNode.DownloadUrl.url() @@ -1324,7 +1526,7 @@ def file_stream( self, file_id: str, max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE, - callback: Callable[..., None] = None, + callback: Optional[Callable[..., None]] = None, encrypt_password: bytes = b"", ) -> Optional[RangeRequestIO]: info = self.download_link(file_id) @@ -1379,7 +1581,7 @@ def _request( ) return resp except Exception as err: - raise AliPCSError("AliOpenAuth._request", cause=err) + raise AliPCSError("AliOpenAuth._request") from err @staticmethod def qrcode_url(sid: str) -> str: From 3d588531d45760bcf53be4e2ad15d472920b3e65 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 17:52:25 +0800 Subject: [PATCH 10/32] New/Update Errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. **AliPCSBaseError** class The base Exception class used for the PCS errors. 2. **AliPCSError(AliPCSBaseError)** class The error returned from alipan server when the client’s request is incorrect or the token is expired. It throw at **AliPCS** class when an error occurs. 3. **DownloadError(AliPCSBaseError)** class An error occurs when downloading action fails. 4. **UploadError(AliPCSBaseError)** class An error occurs when uploading action fails. 5. **RapidUploadError(UploadError)** class An error occurred when rapid uploading action fails. 6. **make_alipcs_error** function Make an AliPCSError instance. 7. **handle_error** function uses the `_error_max_retries` attribute of the wrapped method’s class to retry. --- alipcs_py/alipcs/errors.py | 58 ++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/alipcs_py/alipcs/errors.py b/alipcs_py/alipcs/errors.py index f6ba471..7eeba76 100644 --- a/alipcs_py/alipcs/errors.py +++ b/alipcs_py/alipcs/errors.py @@ -65,32 +65,26 @@ def make_alipcs_error(error_code: str, info: Any = None) -> AliPCSError: return AliPCSError(msg, error_code=error_code) -def assert_ok(func): - """Assert the errno of response is not 0""" - - @wraps(func) - def check(*args, **kwargs): - info = func(*args, **kwargs) - error_code = info.get("code") - - if error_code: - err = make_alipcs_error(error_code, str(info)) - raise err - - return info - - return check - - def handle_error(func): + """Handle error when calling AliPCS API.""" + @wraps(func) - def refresh(*args, **kwargs): + def retry(*args, **kwargs): + self = args[0] + max_retries = getattr(self, "_error_max_retries", 2) code = "This is impossible !!!" - for _ in range(2): - self = args[0] - - info = func(*args, **kwargs) - code = info.get("code") + result = None + for _ in range(max_retries): + result = func(*args, **kwargs) + if not isinstance(result, dict): + return result + + code = result.get("code") + if not code: + return result + + # Error code + # {{{ if code == "AccessTokenInvalid": self.refresh() continue @@ -109,7 +103,7 @@ def refresh(*args, **kwargs): continue elif code == "ParamFlowException": - logger.warning("ParamFlowException, sleep 10s") + logger.warning("ParamFlowException occurs. sleep 10s.") time.sleep(10) continue @@ -117,12 +111,16 @@ def refresh(*args, **kwargs): self._signature = "" continue - elif code == "NotFound.File" and func.__name__ == "meta": - # keep the not found file info for meta api - return info + elif code.startswith("NotFound."): + break + # }}} - return info + # Status code + # {{{ + elif code == "PreHashMatched": # AliPCS.create_file: Pre hash matched. + return result + # }}} - raise make_alipcs_error(code) + raise make_alipcs_error(code, info=result) - return refresh + return retry From 06fd93145cae9c1cc5589774cdd07d9e2a540e32 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 18:53:50 +0800 Subject: [PATCH 11/32] **New/Changed APIs are following:** - `path_traceback` method (**New**) Traceback the path of the file. Return the list of all `PcsFile`s from the file to the top level directory. > *Important*: > The `path` property of the returned `PcsFile` has absolute path. - `meta_by_path` method (**New**) Get the meta of the the path. Can not get the shared files' meta info by their paths. > *Important*: > The `path` property of the returned `PcsFile` is the argument `remotepath`. - `meta` method (**Changed**) Get meta info of the file. > *Important*: > The `path` property of the returned `PcsFile` is only the name of the file. - `get_file` method (**New**) Get the file's info by the given `remotepath` or `file_id` If the `remotepath` is given, the `file_id` will be ignored. > *Important*: > If the `remotepath` is given, the `path` property of the returned `PcsFile` is the `remotepath`. > If the `file_id` is given, the `path` property of the returned `PcsFile` is only the name of the file. - `exists` method (**Changed**) Check whether the file exists. Return True if the file exists and does not in the trash else False. - `exists_in_trash` method (**Changed**) Check whether the file exists in the trash. Return True if the file exists in the trash else False. - `list` method (**Changed**) List files and directories in the given directory (which has the `file_id`). The return items size is limited by the `limit` parameter. If you want to list more, using the returned `next_marker` parameter for next `list` call. > *Important*: > These PcsFile instances' path property is only the name of the file. - `list_iter` method (**Changed**) Iterate all files and directories at the directory (which has the `file_id`). > These returned PcsFile instances' path property is the path from the first sub-directory of the `file_id` to the file name. > e.g. > If the directory (owned `file_id`) has path `level0/`, a sub-directory which of path is > `level0/level1/level2` then its corresponding PcsFile.path is `level1/level2`. - `path` method (**Changed**) Get the pcs file's info by the given absolute `remotepath` > *Important*: > The `path` property of the returned `PcsFile` is the argument `remotepath`. - `list_path` method (**Removed**) - `list_path_iter` method (**Removed**) - `walk` method (**New**) Recursively Walk through the directory tree which has `file_id`. > *Important*: > These PcsFile instances' path property is the path from the first sub-directory of the `file_id` to the file. > e.g. > If the directory (owned `file_id`) has path `level0/`, a sub-directory which of path is > `level0/level1/level2` then its corresponding PcsFile.path is `level1/level2`. - `makedir` method (**Changed**) Make a directory in the `dir_id` directory > *Important*: > The `path` property of the returned `PcsFile` is only the name of the directory. - **makedir_path** method (**Changed**) Make a directory by the absolute `remotedir` path Return the list of all `PcsFile`s from the directory to the top level directory. > *Important*: > The `path` property of the returned `PcsFile` has absolute path. - `rename` method (**Changed**) Rename the file with `file_id` to `name` > *Important*: > The `path` property of the returned `PcsFile` is only the name of the file. - `copy` method (**Changed**) Copy `file_ids[:-1]` to `file_ids[-1]` > *Important*: > The `path` property of the returned `PcsFile` is only the name of the file. - `update_download_url` method (**New**) Update the download url of the `pcs_file` if it is expired. Return a new `PcsFile` with the updated download url. --- alipcs_py/alipcs/api.py | 518 +++++++++++++++++++++++++++++----------- 1 file changed, 376 insertions(+), 142 deletions(-) diff --git a/alipcs_py/alipcs/api.py b/alipcs_py/alipcs/api.py index 47038e7..14de103 100644 --- a/alipcs_py/alipcs/api.py +++ b/alipcs_py/alipcs/api.py @@ -1,9 +1,10 @@ -from typing import Optional, List, Tuple, Dict, Union, DefaultDict, Iterable, Iterator, Callable, IO +from typing import Optional, List, Tuple, Dict, Union, DefaultDict, Iterable, Callable, IO from threading import Lock from collections import defaultdict from copy import deepcopy from functools import partial import logging +import warnings from alipcs_py.alipcs.errors import AliPCSError from alipcs_py.common.io import RangeRequestIO, DEFAULT_MAX_CHUNK_SIZE @@ -20,10 +21,28 @@ class AliPCSApi: - """Aliyundrive PCS Api - - This is the wrapper of `AliPCS`. It parses the content of response of raw - AliPCS requests to some inner data structions. + """Alipan Drive Personal Cloud Service API + + This is the wrapper of `AliPCS` class. It parses the raw content of response of + AliPCS request into the inner data structions. + + Args: + refresh_token (str): The refresh token of the user. + access_token (str, optional): The access token of the user. + token_type (str): The token type. Default is "Bearer". + expire_time (int, optional): The expire time of the token. + user_id (str, optional): The user id of the user. + user_name (str, optional): The user name of the user. + nick_name (str, optional): The nick name of the user. + device_id (str, optional): The device id of the user. + default_drive_id (str, optional): The default drive id of the user. + role (str, optional): The role of the user. + status (str, optional): The status of the user. + error_max_retries (int): The max retries when a client request error occurs. Default is 2. + max_keepalive_connections (int): The max keepalive connections. Default is 50. + max_connections (int): The max number of connections in the pool. Default is 50. + keepalive_expiry (float): The keepalive expiry. Default is 10 * 60 seconds. + connection_max_retries (int): The max retries when a connection error occurs. Default is 2. """ def __init__( @@ -39,6 +58,11 @@ def __init__( default_drive_id: str = "", role: str = "", status: str = "", + error_max_retries: int = 2, + max_keepalive_connections: int = 50, + max_connections: int = 50, + keepalive_expiry: float = 10 * 60, + connection_max_retries: int = 2, ): self._alipcs = AliPCS( refresh_token, @@ -52,6 +76,11 @@ def __init__( default_drive_id=default_drive_id, role=role, status=status, + error_max_retries=error_max_retries, + max_keepalive_connections=max_keepalive_connections, + max_connections=max_connections, + keepalive_expiry=keepalive_expiry, + connection_max_retries=connection_max_retries, ) # The path tree is for user's own files @@ -104,42 +133,124 @@ def role(self) -> str: def status(self) -> str: return self._alipcs.status - def meta(self, *file_ids: str, share_id: str = None) -> List[PcsFile]: - """Meta data of `remotepaths`""" + def path_traceback(self, file_id: str, share_id: Optional[str] = None) -> List[PcsFile]: + """Traceback the path of the file + + Return the list of all `PcsFile`s from the file to the top level directory. + + Important: + The `path` property of the returned `PcsFile` has absolute path. + """ + + try: + info = self._alipcs.path_traceback(file_id, share_id=share_id) + except AliPCSError as err: + if err.error_code == "NotFound.File": + return [] + raise + + pcs_files = [] + for item_info in info["items"][::-1]: + pcs_file = PcsFile.from_(item_info) + pcs_file.path = join_path(pcs_files[-1].path if pcs_files else "/", pcs_file.name) + pcs_files.append(pcs_file) + + pcs_files.reverse() + return pcs_files + + def meta_by_path(self, remotepath: str) -> Optional[PcsFile]: + """Get the meta of the the path + + Can not get the shared files' meta info by their paths. + + Important: + The `path` property of the returned `PcsFile` is the argument `remotepath`. + """ + + assert remotepath.startswith("/"), "Path should start with '/'" + + if remotepath == "/": + return PcsFile.root() - pcs_files = [PcsFile.root() if fid == "root" else None for fid in file_ids] - fids = [fid for fid in file_ids if fid != "root"] + try: + info = self._alipcs.meta_by_path(remotepath) + except AliPCSError as err: + if err.error_code == "NotFound.File": + return None + raise + + pcs_file = PcsFile.from_(info) + pcs_file.path = remotepath + return pcs_file + + def meta(self, file_id: str, share_id: Optional[str] = None) -> Optional[PcsFile]: + """Get meta info of the file + + Important: + The `path` property of the returned `PcsFile` is only the name of the file. + """ + + try: + info = self._alipcs.meta(file_id, share_id=share_id) + except AliPCSError as err: + if err.error_code == "NotFound.File": + return None + raise + + return PcsFile.from_(info) + + def get_file( + self, *, remotepath: str = "", file_id: str = "", share_id: Optional[str] = None + ) -> Optional[PcsFile]: + """Get the file's info by the given `remotepath` or `file_id` - if fids: - info = self._alipcs.meta(*fids, share_id=share_id) - pfs = [PcsFile.from_(v.get("body")) for v in info["responses"]] - j = 0 - for i in range(len(pcs_files)): - if pcs_files[i] is None: - pcs_files[i] = pfs[j] - j += 1 + If the `remotepath` is given, the `file_id` will be ignored. - return [pf for pf in pcs_files if pf is not None] + Important: + If the `remotepath` is given, the `path` property of the returned `PcsFile` is the `remotepath`. + If the `file_id` is given, the `path` property of the returned `PcsFile` is only the name of the file. + """ + + if remotepath: + if share_id: + return self.path(remotepath, share_id=share_id) + else: + return self.meta_by_path(remotepath) + elif file_id: + return self.meta(file_id, share_id=share_id) + else: + raise ValueError("One of `remotepath` and `file_id` must be given") def exists(self, file_id: str) -> bool: - """Check whether `remotepath` exists""" + """Check whether the file exists + + Return True if the file exists and does not in the trash else False. + """ return self._alipcs.exists(file_id) + def exists_in_trash(self, file_id: str) -> bool: + """Check whether the file exists in the trash + + Return True if the file exists in the trash else False. + """ + + return self._alipcs.exists_in_trash(file_id) + def is_file(self, file_id: str) -> bool: - """Check whether `remotepath` is a file""" + """Check whether `file_id` is a file""" return self._alipcs.is_file(file_id) def is_dir(self, file_id: str) -> bool: - """Check whether `remotepath` is a directory""" + """Check whether `file_id` is a directory""" return self._alipcs.is_dir(file_id) def list( self, file_id: str, - share_id: str = None, + share_id: Optional[str] = None, desc: bool = False, name: bool = False, time: bool = False, @@ -154,6 +265,24 @@ def list( List files and directories in the given directory (which has the `file_id`). The return items size is limited by the `limit` parameter. If you want to list more, using the returned `next_marker` parameter for next `list` call. + + Args: + file_id (str): The directory's file id. + share_id (str): The share id if the file_id is in shared link. + desc (bool): Descending order by time. + name (bool): Order by name. + time (bool): Order by time. + size (bool): Order by size. + all (bool): Unknown, just for the request. + limit (int): The number of items to return. + url_expire_sec (int): The download url's expire time. + next_marker (str): The next marker for next list call. + + Returns: + Tuple[List[PcsFile], str]: The list of `PcsFile` and the next marker. + + Important: + These PcsFile instances' path property is only the name of the file. """ info = self._alipcs.list( @@ -169,12 +298,15 @@ def list( next_marker=next_marker, ) next_marker = info["next_marker"] - return [PcsFile.from_(v) for v in info.get("items", [])], next_marker + pcs_files = [] + for v in info.get("items", []): + pcs_files.append(PcsFile.from_(v)) + return pcs_files, next_marker def list_iter( self, file_id: str, - share_id: str = None, + share_id: Optional[str] = None, desc: bool = False, name: bool = False, time: bool = False, @@ -184,15 +316,35 @@ def list_iter( url_expire_sec: int = 14400, recursive: bool = False, include_dir: bool = True, - ) -> Iterator[PcsFile]: + ) -> Iterable[PcsFile]: """Iterate the directory by its `file_id` Iterate all files and directories at the directory (which has the `file_id`). + + Args: + file_id (str): The directory's file id. + share_id (str): The share id if the file_id is in shared link. + desc (bool): Descending order by time. + name (bool): Order by name. + time (bool): Order by time. + size (bool): Order by size. + all (bool): Unknown, just for the request. + limit (int): The number of one request queries. + url_expire_sec (int): The download url's expire time. + recursive (bool): Recursively iterate the directory. + include_dir (bool): Include directory in the result. + + Returns: + Iterable[PcsFile]: The iterator of `PcsFile`. + + Important: + These PcsFile instances' path property is the path from the first sub-directory of the `file_id` to the file name. + e.g. + If the directory (owned `file_id`) has path `level0/`, a sub-directory which of path is + `level0/level1/level2` then its corresponding PcsFile.path is `level1/level2`. """ next_marker = "" - pcs_file = self.meta(file_id, share_id=share_id)[0] - dirname = pcs_file.name while True: pcs_files, next_marker = self.list( file_id, @@ -207,11 +359,13 @@ def list_iter( next_marker=next_marker, ) for pf in pcs_files: - pf.path = join_path(dirname, pf.name) - - if pf.is_dir: + if pf.is_file: + yield pf + else: if include_dir: - yield pf + # The upper recursive call will change the `pf.path`. + # So, we need to deepcopy it. + yield deepcopy(pf) if recursive: for sub_pf in self.list_iter( pf.file_id, @@ -226,95 +380,89 @@ def list_iter( recursive=recursive, include_dir=include_dir, ): - sub_pf.path = join_path(dirname, sub_pf.path) - yield sub_pf - else: - yield pf + sub_pf.path = join_path(pf.path, sub_pf.path) + if sub_pf.is_file: + yield sub_pf + else: + # The upper recursive call will change the `pf.path`. + # So, we need to deepcopy it. + yield deepcopy(sub_pf) if not next_marker: return - def path(self, remotepath: str, share_id: str = None) -> Optional[PcsFile]: - """Get the pcs file's info by the given absolute `remotepath`""" + def path(self, remotepath: str, share_id: Optional[str] = None) -> Optional[PcsFile]: + """Get the pcs file's info by the given absolute `remotepath` + + Important: + The `path` property of the returned `PcsFile` is the argument `remotepath`. + """ + + assert remotepath.startswith("/"), "`remotepath` should start with '/'" if share_id: return self._shared_path_tree[share_id].search(remotepath=remotepath, share_id=share_id) else: return self._path_tree.search(remotepath=remotepath) - def paths(self, *remotepaths: str, share_id: str = None) -> List[Optional[PcsFile]]: - """Get the pcs files' info by the given absolute `remotepaths`""" + def paths(self, *remotepaths: str, share_id: Optional[str] = None) -> List[Optional[PcsFile]]: + """Get the pcs files' info by the given absolute `remotepaths` - return [self.path(rp, share_id=share_id) for rp in remotepaths] + Important: + The `path` property of the returned `PcsFile` is the argument `remotepath`. + """ - def list_path_iter( + return [self.path(remote_path, share_id=share_id) for remote_path in remotepaths] + + def walk( self, - remotepath: str, - file_id: str = None, - share_id: str = None, - desc: bool = False, - name: bool = False, - time: bool = False, - size: bool = False, + file_id: str, + share_id: str = "", all: bool = False, limit: int = 200, url_expire_sec: int = 14400, - recursive: bool = False, - include_dir: bool = True, - ) -> Iterator[PcsFile]: - """Iterate the `remotepath`""" - - if not file_id: - pf = self.path(remotepath, share_id=share_id) - if not pf: - return - file_id = pf.file_id + ) -> Iterable[PcsFile]: + """Recursively Walk through the directory tree which has `file_id` - dirname = posix_path_dirname(remotepath) + Args: + file_id (str): The directory's file id. + share_id (str): The share id if the file_id is in shared link. + all (bool): Unknown, just for the request. + limit (int): The number of one request queries. + url_expire_sec (int): The download url's expire time. + include_dir (bool): Include directory in the result. + + Returns: + Iterable[PcsFile]: The iterator of `PcsFile`. + + Important: + These PcsFile instances' path property is the path from the first sub-directory of the `file_id` to the file. + e.g. + If the directory (owned `file_id`) has path `level0/`, a sub-directory which of path is + `level0/level1/level2` then its corresponding PcsFile.path is `level1/level2`. - for p in self.list_iter( - file_id, - share_id=share_id, - desc=desc, - name=name, - time=time, - size=size, - all=all, - limit=limit, - url_expire_sec=url_expire_sec, - recursive=recursive, - include_dir=include_dir, - ): - p.path = join_path(dirname, p.path) - yield p + """ - def list_path( - self, - remotepath: str, - file_id: str = None, - share_id: str = None, - desc: bool = False, - name: bool = False, - time: bool = False, - size: bool = False, - all: bool = False, - limit: int = 200, - url_expire_sec: int = 14400, - ) -> List[PcsFile]: - return list( - self.list_path_iter( - remotepath, - file_id=file_id, + file_id_to_path = {file_id: ""} + next_marker = "" + while True: + info = self._alipcs.walk( + file_id, share_id=share_id, - desc=desc, - name=name, - time=time, - size=size, all=all, limit=limit, url_expire_sec=url_expire_sec, + next_marker=next_marker, ) - ) + for v in info["items"]: + pcs_file = PcsFile.from_(v) + pcs_file.path = join_path(file_id_to_path[pcs_file.parent_file_id], pcs_file.name) + file_id_to_path[pcs_file.file_id] = pcs_file.path + yield pcs_file + + next_marker = info["next_marker"] + if not next_marker: + return def create_file( self, @@ -400,7 +548,9 @@ def rapid_upload_file( filename, dir_id, size, content_hash=content_hash, proof_code=proof_code, check_name_mode=check_name_mode ) - def upload_slice(self, io: IO, url: str, callback: Callable[[MultipartEncoderMonitor], None] = None) -> None: + def upload_slice( + self, io: IO, url: str, callback: Optional[Callable[[MultipartEncoderMonitor], None]] = None + ) -> None: """Upload an io as a slice callable: the callback for monitoring uploading progress @@ -471,12 +621,27 @@ def search_all( return pcs_files def makedir(self, dir_id: str, name: str) -> PcsFile: + """Make a directory in the `dir_id` directory + + Important: + The `path` property of the returned `PcsFile` is only the name of the directory. + """ + info = self._alipcs.makedir(dir_id, name) return PcsFile.from_(info) - def makedir_path(self, remotedir: str) -> PcsFile: + def makedir_path(self, remotedir: str) -> List[PcsFile]: + """Make a directory by the absolute `remotedir` path + + Return the list of all `PcsFile`s from the directory to the top level directory. + + Important: + The `path` property of the returned `PcsFile` has absolute path. + """ + # Use lock to ignore make a directory twice with _ALI_PCS_API_LOCK: + paths = [] parts = split_posix_path(remotedir) parent = PcsFile.root() for i, part in enumerate(parts): @@ -497,7 +662,10 @@ def makedir_path(self, remotedir: str) -> PcsFile: pf.path = now_dir parent = pf - return parent + paths.append(pf) + + paths.reverse() + return paths def move(self, *file_ids: str) -> List[bool]: """Move `file_ids[:-1]` to `file_ids[-1]`""" @@ -511,6 +679,12 @@ def move(self, *file_ids: str) -> List[bool]: return ["code" not in v["body"] for v in info["responses"]] def rename(self, file_id: str, name: str) -> PcsFile: + """Rename the file with `file_id` to `name` + + Important: + The `path` property of the returned `PcsFile` is only the name of the file. + """ + info = self._alipcs.rename(file_id, name) # Remove node from self._path_tree @@ -519,24 +693,28 @@ def rename(self, file_id: str, name: str) -> PcsFile: return PcsFile.from_(info) def copy(self, *file_ids: str) -> List[PcsFile]: - """Copy `remotepaths[:-1]` to `remotepaths[-1]`""" + """Copy `file_ids[:-1]` to `file_ids[-1]` + + Important: + The `path` property of the returned `PcsFile` is only the name of the file. + """ info = self._alipcs.copy(*file_ids) return [PcsFile.from_(v["body"]) for v in info["responses"]] def remove(self, *file_ids: str) -> List[bool]: - """Remove all `remotepaths`""" + """Remove all `file_ids`""" info = self._alipcs.remove(*file_ids) # Remove nodes from self._path_tree - for file_id in file_ids[:-1]: + for file_id in file_ids: self._path_tree.pop_by_file_id(file_id) return ["code" not in v for v in info["responses"]] def share(self, *file_ids: str, password: str = "", period: int = 0, description: str = "") -> PcsSharedLink: - """Share `remotepaths` to public with a optional password + """Share `file_ids` to public with a optional password Args: period (int): The days for expiring. `0` means no expiring @@ -652,7 +830,7 @@ def file_stream( self, file_id: str, max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE, - callback: Callable[..., None] = None, + callback: Optional[Callable[..., None]] = None, encrypt_password: bytes = b"", ) -> Optional[RangeRequestIO]: """File stream as a normal io""" @@ -667,7 +845,7 @@ def shared_file_stream( share_id: str, expire_duration: int = 10 * 60, max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE, - callback: Callable[..., None] = None, + callback: Optional[Callable[..., None]] = None, encrypt_password: bytes = b"", ) -> Optional[RangeRequestIO]: """Shared file stream as a normal io""" @@ -683,7 +861,7 @@ def shared_file_stream( class AliOpenPCSApi: - """Aliyundrive Open PCS Api + """Alipan drive PCS Open Api This is the wrapper of `AliPCS`. It parses the content of response of raw AliPCS requests to some inner data structions. @@ -704,6 +882,11 @@ def __init__( default_drive_id: str = "", role: str = "", status: str = "", + error_max_retries: int = 2, + max_keepalive_connections: int = 50, + max_connections: int = 50, + keepalive_expiry: float = 10 * 60, + connection_max_retries: int = 2, ): self._aliopenpcs = AliOpenPCS( refresh_token, @@ -719,6 +902,11 @@ def __init__( default_drive_id=default_drive_id, role=role, status=status, + error_max_retries=error_max_retries, + max_keepalive_connections=max_keepalive_connections, + max_connections=max_connections, + keepalive_expiry=keepalive_expiry, + connection_max_retries=connection_max_retries, ) # The path tree is for user's own files @@ -779,22 +967,13 @@ def role(self) -> str: def status(self) -> str: return self._aliopenpcs.status - def meta(self, *file_ids: str, share_id: str = None) -> List[PcsFile]: - """Meta data of `remotepaths`""" - - pcs_files = [PcsFile.root() if fid == "root" else None for fid in file_ids] - fids = [fid for fid in file_ids if fid != "root"] + def meta(self, file_id: str, share_id: Optional[str] = None) -> Optional[PcsFile]: + """Get meta info of the file""" - if fids: - info = self._aliopenpcs.meta(*fids, share_id=share_id) - pfs = [PcsFile.from_(v.get("body")) for v in info["responses"]] - j = 0 - for i in range(len(pcs_files)): - if pcs_files[i] is None: - pcs_files[i] = pfs[j] - j += 1 - - return [pf for pf in pcs_files if pf is not None] + info = self._aliopenpcs.meta(file_id, share_id=share_id) + if info.get("code") == "NotFound.File": + return None + return PcsFile.from_(info) def exists(self, file_id: str) -> bool: """Check whether `remotepath` exists""" @@ -814,7 +993,7 @@ def is_dir(self, file_id: str) -> bool: def list( self, file_id: str, - share_id: str = None, + share_id: Optional[str] = None, desc: bool = False, name: bool = False, time: bool = False, @@ -849,7 +1028,7 @@ def list( def list_iter( self, file_id: str, - share_id: str = None, + share_id: Optional[str] = None, desc: bool = False, name: bool = False, time: bool = False, @@ -859,14 +1038,16 @@ def list_iter( url_expire_sec: int = 14400, recursive: bool = False, include_dir: bool = True, - ) -> Iterator[PcsFile]: + ) -> Iterable[PcsFile]: """Iterate the directory by its `file_id` Iterate all files and directories at the directory (which has the `file_id`). """ next_marker = "" - pcs_file = self.meta(file_id, share_id=share_id)[0] + pcs_file = self.meta(file_id, share_id=share_id) + if pcs_file is None: + return dirname = pcs_file.name while True: pcs_files, next_marker = self.list( @@ -909,7 +1090,7 @@ def list_iter( if not next_marker: return - def path(self, remotepath: str, share_id: str = None) -> Optional[PcsFile]: + def path(self, remotepath: str, share_id: Optional[str] = None) -> Optional[PcsFile]: """Get the pcs file's info by the given absolute `remotepath`""" if share_id: @@ -917,16 +1098,16 @@ def path(self, remotepath: str, share_id: str = None) -> Optional[PcsFile]: else: return self._path_tree.search(remotepath=remotepath) - def paths(self, *remotepaths: str, share_id: str = None) -> List[Optional[PcsFile]]: + def paths(self, *remotepaths: str, share_id: Optional[str] = None) -> List[Optional[PcsFile]]: """Get the pcs files' info by the given absolute `remotepaths`""" - return [self.path(rp, share_id=share_id) for rp in remotepaths] + return [self.path(remote_path, share_id=share_id) for remote_path in remotepaths] def list_path_iter( self, remotepath: str, - file_id: str = None, - share_id: str = None, + file_id: Optional[str] = None, + share_id: Optional[str] = None, desc: bool = False, name: bool = False, time: bool = False, @@ -936,7 +1117,7 @@ def list_path_iter( url_expire_sec: int = 14400, recursive: bool = False, include_dir: bool = True, - ) -> Iterator[PcsFile]: + ) -> Iterable[PcsFile]: """Iterate the `remotepath`""" if not file_id: @@ -966,8 +1147,8 @@ def list_path_iter( def list_path( self, remotepath: str, - file_id: str = None, - share_id: str = None, + file_id: Optional[str] = None, + share_id: Optional[str] = None, desc: bool = False, name: bool = False, time: bool = False, @@ -1016,7 +1197,7 @@ def file_stream( self, file_id: str, max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE, - callback: Callable[..., None] = None, + callback: Optional[Callable[..., None]] = None, encrypt_password: bytes = b"", ) -> Optional[RangeRequestIO]: """File stream as a normal io""" @@ -1027,6 +1208,43 @@ def file_stream( class AliPCSApiMix(AliPCSApi): + """The class mixed with `AliPCSApi` and `AliOpenPCSApi` + + Only following methods are used from AliOpenPCSApi: + - download_link + - update_download_url + - file_stream + Other methods are used from AliPCSApi. + + Args: + web_refresh_token (str): The refresh token from browser. + web_access_token (str, optional): The access token from browser. + web_token_type (str): The token type. Default is "Bearer". + web_expire_time (int, optional): The expire time of the token. + + openapi_refresh_token (str): The refresh token from alipan openapi. + openapi_access_token (str, optional): The access token from alipan openai. + openapi_token_type (str): The token type. Default is "Bearer". + openapi_expire_time (int, optional): The expire time of the token. + client_id (str, optional): The client id of the app for openapi. + client_secret (str, optional): The client secret of the app for openapi. + client_server (str, optional): The client server of the app for openapi to access token. + If `client_id` and `client_secret` are provided, the `client_server` is not needed, vice versa. + + user_id (str, optional): The user id of the user. + user_name (str, optional): The user name of the user. + nick_name (str, optional): The nick name of the user. + device_id (str, optional): The device id of the user. + default_drive_id (str, optional): The default drive id of the user. + role (str, optional): The role of the user. + status (str, optional): The status of the user. + error_max_retries (int): The max retries when a client request error occurs. Default is 2. + max_keepalive_connections (int): The max keepalive connections. Default is 50. + max_connections (int): The max number of connections in the pool. Default is 50. + keepalive_expiry (float): The keepalive expiry. Default is 10 * 60 seconds. + connection_max_retries (int): The max retries when a connection error occurs. Default is 2. + """ + def __init__( self, web_refresh_token: str, @@ -1047,6 +1265,11 @@ def __init__( default_drive_id: str = "", role: str = "", status: str = "", + error_max_retries: int = 2, + max_keepalive_connections: int = 50, + max_connections: int = 50, + keepalive_expiry: float = 10 * 60, + connection_max_retries: int = 2, ): super().__init__( refresh_token=web_refresh_token, @@ -1060,6 +1283,11 @@ def __init__( default_drive_id=default_drive_id, role=role, status=status, + error_max_retries=error_max_retries, + max_keepalive_connections=max_keepalive_connections, + max_connections=max_connections, + keepalive_expiry=keepalive_expiry, + connection_max_retries=connection_max_retries, ) self._aliopenpcsapi: Optional[AliOpenPCSApi] = None @@ -1078,6 +1306,11 @@ def __init__( default_drive_id=default_drive_id, role=role, status=status, + error_max_retries=error_max_retries, + max_keepalive_connections=max_keepalive_connections, + max_connections=max_connections, + keepalive_expiry=keepalive_expiry, + connection_max_retries=connection_max_retries, ) def download_link(self, file_id: str) -> Optional[PcsDownloadUrl]: @@ -1105,7 +1338,7 @@ def file_stream( self, file_id: str, max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE, - callback: Callable[..., None] = None, + callback: Optional[Callable[..., None]] = None, encrypt_password: bytes = b"", ) -> Optional[RangeRequestIO]: """File stream as a normal io""" @@ -1194,12 +1427,11 @@ def _add(self, node: _Node): def _pop(self, file_id: str) -> Optional[_Node]: """Pop a node from self._file_id_to_node""" - try: - return self._file_id_to_node.pop(file_id) - except KeyError: - return None + return self._file_id_to_node.pop(file_id, None) - def search(self, remotepath: str = "", topdown: Iterable[str] = [], share_id: str = None) -> Optional[PcsFile]: + def search( + self, remotepath: str = "", topdown: Iterable[str] = [], share_id: Optional[str] = None + ) -> Optional[PcsFile]: """Search the PcsFile which has remote path as `remotepath` or has the tree path `topdown` """ @@ -1214,7 +1446,9 @@ def search(self, remotepath: str = "", topdown: Iterable[str] = [], share_id: st else: return None - def _dfs(self, topdown: List[str], root: _Node, pull: bool = True, share_id: str = None) -> Optional[_Node]: + def _dfs( + self, topdown: List[str], root: _Node, pull: bool = True, share_id: Optional[str] = None + ) -> Optional[_Node]: """Search a node with the path `topdown` using depth first search""" if not topdown: @@ -1253,7 +1487,7 @@ def pop(self, remotepath: str = "", topdown: Iterable[str] = []) -> Optional[Pcs parts = list(topdown) dest = parts[-1] parent = parts[:-1] - assert len(parent) > 0, "NO pop root" + assert len(parent) > 0, "Can not pop root" node = self._dfs(list(parent), self.root, pull=False) if node: From abd88b9e691fc2697658107074f9255d5acba7a1 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 18:55:05 +0800 Subject: [PATCH 12/32] Fix typos --- alipcs_py/alipcs/pcs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/alipcs_py/alipcs/pcs.py b/alipcs_py/alipcs/pcs.py index 5023175..d9c0b82 100644 --- a/alipcs_py/alipcs/pcs.py +++ b/alipcs_py/alipcs/pcs.py @@ -88,9 +88,9 @@ def url(self) -> str: class AliPCS: - """Aliyun Drive Personal Cloud Service Raw API + """Alipan Drive Personal Cloud Service Raw API - The core class is used to interact with Aliyun Drive Personal Cloud Service. + The core class is used to interact with Alipan Drive Personal Cloud Service. It provides the basic operations of the service and handles the raw requests and responses. An `AliPCSError` error will be raised if the code of the response occurs. @@ -1160,9 +1160,9 @@ def url(self) -> str: class AliOpenPCS: - """Aliyun Drive Personal Cloud Service Raw Open API + """Alipan Drive Personal Cloud Service Raw Open API - The core class is used to interact with Aliyun Drive Personal Cloud Service with open apis. + The core class is used to interact with Alipan Drive Personal Cloud Service with open apis. It provides the basic operations of the service and handles the raw requests and responses. An `AliPCSError` error will be raised if the code of the response occurs. From 7b37ce9e5d0c395d20ba5272f1ee1ee0076f26d3 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 18:56:30 +0800 Subject: [PATCH 13/32] Use `get_file` instead of `path` --- alipcs_py/commands/cat.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/alipcs_py/commands/cat.py b/alipcs_py/commands/cat.py index 1078ee8..9ccea2f 100644 --- a/alipcs_py/commands/cat.py +++ b/alipcs_py/commands/cat.py @@ -12,8 +12,7 @@ def cat( encoding: Optional[str] = None, encrypt_password: bytes = b"", ): - pcs_file = api.path(remotepath) - + pcs_file = api.get_file(remotepath=remotepath) if not pcs_file: return From fbb7a9be1ebee2732d2a3a318e0ff55cc8a39074 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 18:56:58 +0800 Subject: [PATCH 14/32] Ignore existed directories when creating them --- alipcs_py/commands/crypto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alipcs_py/commands/crypto.py b/alipcs_py/commands/crypto.py index 83e3392..be9f80d 100644 --- a/alipcs_py/commands/crypto.py +++ b/alipcs_py/commands/crypto.py @@ -13,7 +13,7 @@ def decrypt_file(from_encrypted: PathLike, to_decrypted: PathLike, encrypt_passw dpath = Path(to_decrypted) dir_ = dpath.parent if not dir_.exists(): - dir_.mkdir(parents=True) + dir_.mkdir(parents=True, exist_ok=True) with dpath.open("wb") as dfd: while True: From 86fd30b4b97e44535764139993215acf4dce258e Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 19:17:09 +0800 Subject: [PATCH 15/32] Update/Change - `DownloadParams` class (**Removed**) We remove the `DownloadParams` instead of using arguments for function calling. - `download_file` function (**Changed**) `download_file` downloads one remote file to one local directory. Raise any error occurred. So giving the upper level caller to handle errors. - `download` function (**Changed**) `download` function downloads any number of remote files/directory to one local directory. It uses a `ThreadPoolExecutor` to download files concurrently and raise the exception if any error occurred. --- alipcs_py/commands/download.py | 319 ++++++++++++++++++++++----------- 1 file changed, 217 insertions(+), 102 deletions(-) diff --git a/alipcs_py/commands/download.py b/alipcs_py/commands/download.py index d7a5071..2741c30 100644 --- a/alipcs_py/commands/download.py +++ b/alipcs_py/commands/download.py @@ -1,5 +1,5 @@ -from typing import Iterable, Optional, List, Tuple -from types import SimpleNamespace +from typing import Iterable, Optional, List, Sequence, Tuple, Union +from concurrent.futures import ThreadPoolExecutor, as_completed from enum import Enum from pathlib import Path import os @@ -9,9 +9,9 @@ import random from alipcs_py.alipcs import AliPCSApi, PcsFile -from alipcs_py.alipcs.errors import AliPCSError +from alipcs_py.alipcs.errors import AliPCSError, DownloadError from alipcs_py.alipcs.pcs import PCS_UA -from alipcs_py.common.concurrent import Executor +from alipcs_py.common.path import PathType from alipcs_py.utils import human_size_to_int from alipcs_py.common import constant from alipcs_py.common.io import RangeRequestIO, to_decryptio, DecryptIO, READ_SIZE @@ -41,16 +41,6 @@ MAX_CHUNK_SIZE = 50 * constant.OneM -class DownloadParams(SimpleNamespace): - concurrency: int = DEFAULT_CONCURRENCY - chunk_size: str = DEFAULT_CHUNK_SIZE - quiet: bool = False - retries: int = 2 - - -DEFAULT_DOWNLOADPARAMS = DownloadParams() - - class Downloader(Enum): me = "me" aget_py = "aget" # https://github.com/PeterDing/aget @@ -69,7 +59,10 @@ def download( self, url: str, localpath: str, - downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS, + concurrency: int = DEFAULT_CONCURRENCY, + chunk_size: Union[str, int] = DEFAULT_CHUNK_SIZE, + show_progress: bool = False, + max_retries: int = 2, out_cmd: bool = False, encrypt_password: bytes = b"", ): @@ -83,24 +76,47 @@ def download( self._me_download( url, localpath_tmp, - downloadparams=downloadparams, + chunk_size=chunk_size, + show_progress=show_progress, + max_retries=max_retries, encrypt_password=encrypt_password, ) shutil.move(localpath_tmp, localpath) return elif self == Downloader.aget_py: - cmd = self._aget_py_cmd(url, localpath_tmp, downloadparams) + cmd = self._aget_py_cmd( + url, + localpath_tmp, + concurrency=concurrency, + chunk_size=chunk_size, + show_progress=show_progress, + max_retries=max_retries, + ) elif self == Downloader.aget_rs: - cmd = self._aget_rs_cmd(url, localpath_tmp, downloadparams) + cmd = self._aget_rs_cmd( + url, + localpath_tmp, + concurrency=concurrency, + chunk_size=chunk_size, + show_progress=show_progress, + max_retries=max_retries, + ) else: # elif self == Downloader.aria2: - cmd = self._aria2_cmd(url, localpath_tmp, downloadparams) + cmd = self._aria2_cmd( + url, + localpath_tmp, + concurrency=concurrency, + chunk_size=chunk_size, + show_progress=show_progress, + max_retries=max_retries, + ) # Print out command if out_cmd: _print(" ".join((repr(c) for c in cmd))) return - returncode = self.spawn(cmd, downloadparams.quiet) + returncode = self.spawn(cmd, show_progress=show_progress) logger.debug("`download`: cmd returncode: %s", returncode) @@ -121,15 +137,17 @@ def download( return shutil.move(localpath_tmp, localpath) - def spawn(self, cmd: List[str], quiet: bool = False): - child = subprocess.run(cmd, stdout=subprocess.DEVNULL if quiet else None) + def spawn(self, cmd: List[str], show_progress: bool = False): + child = subprocess.run(cmd, stdout=subprocess.DEVNULL if not show_progress else None) return child.returncode def _me_download( self, url: str, localpath: str, - downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS, + chunk_size: Union[str, int] = DEFAULT_CHUNK_SIZE, + show_progress: bool = False, + max_retries: int = 2, encrypt_password: bytes = b"", ): headers = { @@ -139,7 +157,7 @@ def _me_download( } task_id: Optional[TaskID] = None - if not downloadparams.quiet: + if show_progress: init_progress_bar() task_id = _progress.add_task("MeDownloader", start=False, title=localpath) @@ -153,12 +171,13 @@ def monitor_callback(offset: int): def except_callback(err): reset_progress_task(task_id) - chunk_size_int = human_size_to_int(downloadparams.chunk_size) + if isinstance(chunk_size, str): + chunk_size = human_size_to_int(chunk_size) io = RangeRequestIO( "GET", url, headers=headers, - max_chunk_size=chunk_size_int, + max_chunk_size=chunk_size, callback=monitor_callback, encrypt_password=encrypt_password, ) @@ -170,9 +189,9 @@ def except_callback(err): meDownloader = MeDownloader( io, - localpath=Path(localpath), + localpath=localpath, continue_=True, - retries=downloadparams.retries, + max_retries=max_retries, done_callback=done_callback, except_callback=except_callback, ) @@ -182,7 +201,10 @@ def _aget_py_cmd( self, url: str, localpath: str, - downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS, + concurrency: int = DEFAULT_CONCURRENCY, + chunk_size: Union[str, int] = DEFAULT_CHUNK_SIZE, + show_progress: bool = False, + max_retries: int = 2, ): cmd = [ self.which(), @@ -196,17 +218,22 @@ def _aget_py_cmd( "-H", "Referer: https://www.aliyundrive.com/", "-s", - str(downloadparams.concurrency), + str(concurrency), "-k", - downloadparams.chunk_size, + chunk_size, ] + if not show_progress: + cmd.append("-q") return cmd def _aget_rs_cmd( self, url: str, localpath: str, - downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS, + concurrency: int = DEFAULT_CONCURRENCY, + chunk_size: Union[str, int] = DEFAULT_CHUNK_SIZE, + show_progress: bool = False, + max_retries: int = 2, ): cmd = [ self.which(), @@ -220,17 +247,25 @@ def _aget_rs_cmd( "-H", "Referer: https://www.aliyundrive.com/", "-s", - str(downloadparams.concurrency), + str(concurrency), "-k", - downloadparams.chunk_size, + chunk_size, ] + if not show_progress: + cmd.append("--quiet") + if max_retries > 0: + cmd.append("--retries") + cmd.append(str(max_retries)) return cmd def _aria2_cmd( self, url: str, localpath: str, - downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS, + concurrency: int = DEFAULT_CONCURRENCY, + chunk_size: Union[str, int] = DEFAULT_CHUNK_SIZE, + show_progress: bool = False, + max_retries: int = 2, ): directory, filename = os.path.split(localpath) cmd = [ @@ -247,11 +282,13 @@ def _aria2_cmd( "--header", "Referer: https://www.aliyundrive.com/", "-s", - str(downloadparams.concurrency), + str(concurrency), "-k", - downloadparams.chunk_size, + chunk_size, url, ] + if not show_progress: + cmd.append("--quiet") return cmd @@ -260,47 +297,85 @@ def _aria2_cmd( def download_file( api: AliPCSApi, - pcs_file: PcsFile, - localdir: str, - share_id: str = None, - downloader: Downloader = DEFAULT_DOWNLOADER, - downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS, + remote_file: Union[str, PcsFile], + localdir: PathType = ".", + share_id: Optional[str] = None, + downloader: Union[str, Downloader] = DEFAULT_DOWNLOADER, + concurrency: int = DEFAULT_CONCURRENCY, + chunk_size: Union[str, int] = DEFAULT_CHUNK_SIZE, + show_progress: bool = False, + max_retries: int = 2, out_cmd: bool = False, encrypt_password: bytes = b"", -): - quiet = downloadparams.quiet - localpath = Path(localdir) / pcs_file.name +) -> None: + """Download a `remote_file` to the `localdir` + + Raise the exception if any error occurred. + + Args: + api (AliPCSApi): AliPCSApi instance. + remote_file (str | PcsFile): The remote file to download. + localdir (str | PathLike | Path, optional): The local directory to save file. Defaults to ".". + share_id (str, optional): The share_id of file. Defaults to None. + downloader (str, Downloader, optional): The downloader(or its name) to download file. Defaults to DEFAULT_DOWNLOADER. + concurrency (int, optional): The number of concurrent downloads. Defaults to DEFAULT_CONCURRENCY. + chunk_size (str | int, optional): The chunk size of each download. Defaults to DEFAULT_CHUNK_SIZE. + show_progress (bool, optional): Whether show progress bar. Defaults to False. + max_retries (int, optional): The max retries of download. Defaults to 2. + out_cmd (bool, optional): Whether print out the command. Defaults to False. + encrypt_password (bytes, optional): The password to decrypt the file. Defaults to b"". + """ + + if isinstance(downloader, str): + downloader = getattr(Downloader, downloader) + assert isinstance(downloader, Downloader) # For linters + + if isinstance(remote_file, str): + remote_pcs_file = api.get_file(remotepath=remote_file) + if remote_pcs_file is None: + raise ValueError(f"Remote file `{remote_file}` does not exists.") + else: + remote_pcs_file = remote_file + + localpath = Path(localdir) / remote_pcs_file.name # Make sure parent directory existed if not localpath.parent.exists(): - localpath.parent.mkdir(parents=True) + localpath.parent.mkdir(parents=True, exist_ok=True) if not out_cmd and localpath.exists(): - if not quiet: + if not show_progress: print(f"[yellow]{localpath}[/yellow] is ready existed.") return - if not quiet and downloader != Downloader.me: - print(f"[italic blue]Download[/italic blue]: {pcs_file.path or pcs_file.name} to {localpath}") + if not show_progress and downloader != Downloader.me: + print(f"[italic blue]Download[/italic blue]: {remote_pcs_file.path or remote_pcs_file.name} to {localpath}") - download_url: Optional[str] if share_id: + shared_pcs_file_id = remote_pcs_file.file_id + shared_pcs_filename = remote_pcs_file.name remote_temp_dir = "/__alipcs_py_temp__" - pcs_temp_dir = api.path(remote_temp_dir) or api.makedir_path(remote_temp_dir) - pcs_file = api.transfer_shared_files([pcs_file.file_id], pcs_temp_dir.file_id, share_id)[0] - + pcs_temp_dir = api.path(remote_temp_dir) or api.makedir_path(remote_temp_dir)[0] + pf = api.transfer_shared_files([shared_pcs_file_id], pcs_temp_dir.file_id, share_id)[0] + target_file_id = pf.file_id while True: - pcs_file = api.meta(pcs_file.file_id)[0] - if pcs_file.download_url: - break - time.sleep(2) + pfs = api.search_all(shared_pcs_filename) + for pf_ in pfs: + if pf_.file_id == target_file_id: + remote_pcs_file = pf_ + break + else: + time.sleep(2) + continue - if not pcs_file or pcs_file.is_dir: + break + + if not remote_pcs_file or remote_pcs_file.is_dir: return while True: try: - pcs_file = api.update_download_url(pcs_file) + remote_pcs_file = api.update_download_url(remote_pcs_file) break except AliPCSError as err: if err.error_code == "TooManyRequests": @@ -308,37 +383,40 @@ def download_file( continue raise err - download_url = pcs_file.download_url - + download_url = remote_pcs_file.download_url assert download_url try: downloader.download( download_url, str(localpath), - downloadparams=downloadparams, + concurrency=concurrency, + chunk_size=chunk_size, + show_progress=show_progress, + max_retries=max_retries, out_cmd=out_cmd, encrypt_password=encrypt_password, ) - except Exception as err: - logger.error("`download_file` fails: error: %s", err) - if not quiet: - print(f"[red]ERROR[/red]: `{pcs_file.path or pcs_file.name}` download fails.") + except Exception as origin_err: + msg = f'Download "{remote_pcs_file.path}" (file_id = "{remote_pcs_file.file_id}") to "{localpath}" failed. error: {origin_err}' + logger.debug(msg) + err = DownloadError(msg, remote_pcs_file=remote_pcs_file, localdir=str(localdir)) + raise err from origin_err if share_id: - api.remove(pcs_file.file_id) + api.remove(remote_pcs_file.file_id) def walk_remote_paths( api: AliPCSApi, pcs_files: List[PcsFile], - localdir: str, - share_id: str = None, + localdir: PathType, + share_id: Optional[str] = None, sifters: List[Sifter] = [], recursive: bool = False, from_index: int = 0, deep: int = 0, -) -> Iterable[Tuple[PcsFile, str]]: +) -> Iterable[Tuple[PcsFile, PathType]]: pcs_files = [pf for pf in sift(pcs_files, sifters, recursive=recursive)] for pf in pcs_files: if pf.is_file: @@ -347,15 +425,15 @@ def walk_remote_paths( if deep > 0 and not recursive: continue - _localdir = Path(localdir) / pf.name + localdir_ = Path(localdir) / pf.name for pcs_file in api.list_iter(pf.file_id, share_id=share_id): if pcs_file.is_file: - yield pcs_file, str(_localdir) + yield pcs_file, localdir_ else: yield from walk_remote_paths( api, [pcs_file], - str(_localdir), + localdir_, share_id=share_id, sifters=sifters, recursive=recursive, @@ -366,58 +444,86 @@ def walk_remote_paths( def download( api: AliPCSApi, - remotepaths: List[str], - file_ids: List[str], - localdir: str, - share_id: str = None, + remotepaths: Sequence[Union[str, PcsFile]] = [], + file_ids: List[str] = [], + localdir: PathType = ".", + share_id: Optional[str] = None, sifters: List[Sifter] = [], recursive: bool = False, from_index: int = 0, - downloader: Downloader = DEFAULT_DOWNLOADER, - downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS, + downloader: Union[str, Downloader] = DEFAULT_DOWNLOADER, + concurrency: int = DEFAULT_CONCURRENCY, + chunk_size: Union[str, int] = DEFAULT_CHUNK_SIZE, + show_progress: bool = False, + max_retries: int = 2, out_cmd: bool = False, encrypt_password: bytes = b"", -): - """Download `remotepaths` to the `localdir` +) -> None: + """Download files with their `remotepaths` and `file_ids` to the `localdir` + + Use a `ThreadPoolExecutor` to download files concurrently and raise the exception if any error occurred. Args: - `from_index` (int): The start index of downloading entries from EACH remote directory + api (AliPCSApi): AliPCSApi instance. + remotepaths (List[Union[str, PcsFile]], optional): The remotepaths of files or directories to download. + file_ids (List[str], optional): The file_ids of files or directories to download. Defaults to []. + localdir (str | PathLike | Path, optional): The local directory to save files. Defaults to ".". + share_id (str, optional): The share_id of files. Defaults to None. + sifters (List[Sifter], optional): The sifters to filter files. Defaults to []. + recursive (bool, optional): Whether download files recursively. Defaults to False. + from_index (int, optional): The index of the first file to download. Defaults to 0. + downloader (str, Downloader, optional): The downloader(or its name) to download files. Defaults to Downloader.me. + concurrency (int, optional): The number of concurrent downloads. Defaults to DEFAULT_CONCURRENCY. + chunk_size (str | int, optional): The chunk size of each download. Defaults to DEFAULT_CHUNK_SIZE. + show_progress (bool, optional): Whether show progress bar. Defaults to False. + max_retries (int, optional): The max retries of download. Defaults to 2. + out_cmd (bool, optional): Whether print out the command. Defaults to False. + encrypt_password (bytes, optional): The password to decrypt the file. Defaults to b"". """ logger.debug( - "`download`: sifters: %s, recursive: %s, from_index: %s, " - "downloader: %s, downloadparams: %s, out_cmd: %s, has encrypt_password: %s", + "download: remotepaths=%s, file_ids=%s, localdir=%s, share_id=%s, sifters=%s, recursive=%s, from_index=%s, downloader=%s, concurrency=%s, chunk_size=%s, quiet=%s, max_retries=%s, out_cmd=%s, encrypt_password=%s", + [rp.path if isinstance(rp, PcsFile) else rp for rp in remotepaths], + file_ids, + localdir, + share_id, sifters, recursive, from_index, downloader, - downloadparams, + concurrency, + chunk_size, + show_progress, + max_retries, out_cmd, - bool(encrypt_password), + encrypt_password, ) - quiet = downloadparams.quiet + assert len(remotepaths) + len(file_ids) > 0, "No remotepaths or file_ids to download." pcs_files = [] for rp in remotepaths: - pf = api.path(rp, share_id=share_id) + if isinstance(rp, PcsFile): + pcs_files.append(rp) + continue + pf = api.get_file(remotepath=rp, share_id=share_id) if pf is None: - if not quiet: - print(f"[yellow]WARNING[/yellow]: `{rp}` does not exist.") + if not show_progress: + print(f"[yellow]WARNING[/yellow]: `remotepath={rp}` does not exist.") continue pcs_files.append(pf) for file_id in file_ids: - info = api.meta(file_id, share_id=share_id) - if len(info) == 0: - if not quiet: - print(f"[yellow]WARNING[/yellow]: `{file_id}` does not exist.") + pf = api.get_file(file_id=file_id, share_id=share_id) + if pf is None: + if not show_progress: + print(f"[yellow]WARNING[/yellow]: `{file_id=}` does not exist.") continue - pcs_files.append(info[0]) + pcs_files.append(pf) - using_me_downloader = downloader == Downloader.me - with Executor(downloadparams.concurrency if using_me_downloader else 1) as executor: - for pf, _localdir in walk_remote_paths( + futures = [] + with ThreadPoolExecutor(concurrency) as executor: + for pf, localdir_ in walk_remote_paths( api, pcs_files, localdir, @@ -426,17 +532,26 @@ def download( recursive=recursive, from_index=from_index, ): - executor.submit( + fut = executor.submit( download_file, api, pf, - _localdir, + localdir_, share_id=share_id, downloader=downloader, - downloadparams=downloadparams, + concurrency=concurrency, + chunk_size=chunk_size, + show_progress=show_progress, + max_retries=max_retries, out_cmd=out_cmd, encrypt_password=encrypt_password, ) + futures.append(fut) + + # Wait for all futures done + for fut in as_completed(futures): + # Throw the exception if the future has exception + fut.result() - if not quiet: + if not show_progress: _progress.stop() From b140d409acf88dca9ad69cfae33dd07352a9c5f9 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 20:03:38 +0800 Subject: [PATCH 16/32] Update/Change - `UploadType` class (**Removed**) Alipan.com only support to upload a file through uploading slice parts one by one. So, the class is not needed. - `upload_file` function (**Changed**) Upload a file from one local file ( `from_to[0]`) to remote ( `from_to[1]`). First try to rapid upload, if failed, then upload file's slices. Raise exception if any error occurs. - `upload` function (Changed) Upload files in `from_to_list` to Alipan Drive. Use a `ThreadPoolExecutor` to upload files concurrently. Raise exception if any error occurs. --- alipcs_py/commands/upload.py | 673 +++++++++++------------------------ 1 file changed, 213 insertions(+), 460 deletions(-) diff --git a/alipcs_py/commands/upload.py b/alipcs_py/commands/upload.py index 2e109aa..0858b3e 100644 --- a/alipcs_py/commands/upload.py +++ b/alipcs_py/commands/upload.py @@ -1,52 +1,32 @@ -from hashlib import sha1 -from typing import Callable, Optional, List, Tuple, IO - +from typing import Callable, Optional, List, Sequence, Tuple, IO, Union import os import time -import functools import math +from hashlib import sha1 from io import BytesIO -from enum import Enum from pathlib import Path -from threading import Semaphore from concurrent.futures import ThreadPoolExecutor, as_completed -from alipcs_py.alipcs.errors import AliPCSError +from alipcs_py.alipcs.errors import AliPCSError, RapidUploadError, UploadError from alipcs_py.alipcs import AliPCSApi, FromTo from alipcs_py.alipcs.pcs import CheckNameMode from alipcs_py.common import constant -from alipcs_py.common.path import ( - is_file, - exists, - posix_path_basename, - posix_path_dirname, - walk, -) +from alipcs_py.common.path import PathType, is_file, exists, posix_path_basename, posix_path_dirname from alipcs_py.common.event import KeyHandler, KeyboardMonitor from alipcs_py.common.constant import CPU_NUM -from alipcs_py.common.concurrent import sure_release, retry -from alipcs_py.common.progress_bar import ( - _progress, - progress_task_exists, - remove_progress_task, - reset_progress_task, -) +from alipcs_py.common.concurrent import retry +from alipcs_py.common.progress_bar import _progress, progress_task_exists, remove_progress_task, reset_progress_task from alipcs_py.common.crypto import calc_sha1, calc_proof_code -from alipcs_py.common.io import ( - total_len, - EncryptType, - reset_encrypt_io, -) +from alipcs_py.common.io import total_len, EncryptType, reset_encrypt_io from alipcs_py.commands.log import get_logger from requests_toolbelt import MultipartEncoderMonitor from rich.progress import TaskID -from rich.table import Table -from rich.box import SIMPLE -from rich.text import Text from rich import print +from alipcs_py.utils import human_size_to_int + logger = get_logger(__name__) # If slice size >= 100M, the rate of uploading will be much lower. @@ -54,8 +34,6 @@ UPLOAD_STOP = False -_rapiduploadinfo_file: Optional[str] = None - def _wait_start(): while True: @@ -88,107 +66,126 @@ def adjust_slice_size(slice_size: int, io_len: int) -> int: return slice_size -def to_remotepath(sub_path: str, remotedir: str) -> str: - return (Path(remotedir) / sub_path).as_posix() - - -def from_tos(localpaths: List[str], remotedir: str) -> List[FromTo]: - """Find all localpaths and their corresponded remotepath""" +def from_tos(localpaths: Sequence[PathType], remotedir: str) -> List[FromTo]: + """Recursively find all localpaths and their corresponded remotepath""" - ft: List[FromTo] = [] + ft: List[FromTo[Path, str]] = [] for localpath in localpaths: - if not exists(localpath): + localpath = Path(localpath).resolve() + if not localpath.exists(): continue - if is_file(localpath): - remotepath = to_remotepath(os.path.basename(localpath), remotedir) - ft.append(FromTo(localpath, remotepath)) + if localpath.is_file(): + remotepath = Path(remotedir, localpath.name).as_posix() + ft.append((localpath, remotepath)) else: - parents_num = max(len(Path(localpath).parts) - 1, 0) - for sub_path in walk(localpath): - relative_path = Path(*Path(sub_path).parts[parents_num:]).as_posix() - remotepath = to_remotepath(relative_path, remotedir) - ft.append(FromTo(sub_path, remotepath)) + parent_num = len(localpath.parent.parts) + for root, _, filenames in os.walk(localpath): + for filename in filenames: + sub_path = Path(root, filename) + relative_path = Path(*Path(root).parts[parent_num:], filename).as_posix() + remotepath = Path(remotedir, relative_path).as_posix() + ft.append((sub_path, remotepath)) return ft -class UploadType(Enum): - """Upload Type - - One: Upload the slices of one file concurrently - Many: Upload files concurrently - """ - - One = 1 - Many = 2 - - -# remotedir must be a directory def upload( api: AliPCSApi, - from_to_list: List[FromTo], - upload_type: UploadType = UploadType.One, - check_name_mode: CheckNameMode = "auto_rename", + from_to_list: List[FromTo[PathType, str]], + check_name_mode: CheckNameMode = "overwrite", encrypt_password: bytes = b"", encrypt_type: EncryptType = EncryptType.No, max_workers: int = CPU_NUM, - slice_size: int = DEFAULT_SLICE_SIZE, - show_progress: bool = True, - rapiduploadinfo_file: Optional[str] = None, - user_id: Optional[str] = None, - user_name: Optional[str] = None, -): - """Upload from_tos + max_retries: int = 3, + slice_size: Union[str, int] = DEFAULT_SLICE_SIZE, + only_use_rapid_upload: bool = False, + show_progress: bool = False, +) -> None: + r"""Upload files in `from_to_list` to Alipan Drive + + Use a `ThreadPoolExecutor` to upload files concurrently. + + Raise exception if any error occurs. Args: - upload_type (UploadType): the way of uploading. - max_workers (int): The number of concurrent workers. - slice_size (int): The size of slice for uploading slices. - ignore_existing (bool): Ignoring these localpath which of remotepath exist. - show_progress (bool): Show uploading progress. - check_name_mode(str): - 'overwrite' (直接覆盖，以后多版本有用) - 'auto_rename' (自动换一个随机名称) - 'refuse' (不会创建，告诉你已经存在) - 'ignore' (会创建重名的) + api (AliPCSApi): AliPCSApi instance. + from_to_list (List[FromTo[PathType, str]]): List of FromTo instances which decide the local path needed to upload and the remote path to upload to. + check_name_mode (CheckNameMode, optional): CheckNameMode. Defaults to "overwrite". + encrypt_password (bytes, optional): Encrypt password. Defaults to b"". + encrypt_type (EncryptType, optional): Encrypt type. Defaults to EncryptType.No. + max_workers (int, optional): Max workers. Defaults to the number of CPUs. + max_retries (int, optional): Max retries. Defaults to 3. + slice_size (Union[str, int], optional): Slice size. Defaults to DEFAULT_SLICE_SIZE. + only_use_rapid_upload (bool, optional): Only use rapid upload. If rapid upload fails, raise exception. Defaults to False. + show_progress (bool, optional): Show progress. Defaults to False. + + Examples: + - Upload one file to one remote directory + + ```python + >>> from alipcs_py.alipcs import AliPCSApi + >>> from alipcs_py.commands.upload import upload, from_tos + >>> api = AliPCSApi(...) + >>> remotedir = "/remote/dir" + >>> localpath = "/local/file" + >>> from_to_list = from_tos([localpath], remotedir) + >>> upload(api, from_to_list) + ``` + + - Upload multiple files and directories recursively to one remote directory + + ```python + >>> from alipcs_py.alipcs import AliPCSApi + >>> from alipcs_py.commands.upload import upload, from_tos + >>> api = AliPCSApi(...) + >>> remotedir = "/remote/dir" + >>> target_paths = ['/local/file1', '/local/file2', '/local/dir1', '/local/dir2'] + >>> from_to_list = from_tos(target_paths, remotedir) + >>> upload(api, from_to_list) + ``` """ logger.debug( - "======== Uploading start ========\n-> UploadType: %s\n-> Size of from_to_list: %s", - upload_type, + "======== Uploading start ========\n-> Size of from_to_list: %s", len(from_to_list), ) - global _rapiduploadinfo_file - if _rapiduploadinfo_file is None: - _rapiduploadinfo_file = rapiduploadinfo_file - - if upload_type == UploadType.One: - upload_one_by_one( - api, - from_to_list, - check_name_mode, - max_workers=max_workers, - encrypt_password=encrypt_password, - encrypt_type=encrypt_type, - slice_size=slice_size, - show_progress=show_progress, - user_id=user_id, - user_name=user_name, - ) - elif upload_type == UploadType.Many: - upload_many( - api, - from_to_list, - check_name_mode, - max_workers=max_workers, - encrypt_password=encrypt_password, - encrypt_type=encrypt_type, - slice_size=slice_size, - show_progress=show_progress, - user_id=user_id, - user_name=user_name, - ) + futures = [] + with ThreadPoolExecutor(max_workers=max_workers) as executor: + for idx, from_to in enumerate(from_to_list): + task_id = None + if show_progress: + task_id = _progress.add_task("upload", start=False, title=from_to[0]) + + logger.debug("`upload_many`: Upload: index: %s, task_id: %s", idx, task_id) + + retry_upload_file = retry( + max_retries, + except_callback=lambda err, fail_count: logger.warning( + "`upload_file`: fails: error: %s, fail_count: %s", + err, + fail_count, + exc_info=err, + ), + )(upload_file) + + fut = executor.submit( + retry_upload_file, + api, + from_to, + check_name_mode, + encrypt_password=encrypt_password, + encrypt_type=encrypt_type, + slice_size=slice_size, + only_use_rapid_upload=only_use_rapid_upload, + task_id=task_id, + ) + futures.append(fut) + + # Wait for all futures done + for fut in as_completed(futures): + # Raise the exception if the result of the future is an exception + fut.result() def _need_to_upload(api: AliPCSApi, remotepath: str, check_name_mode: CheckNameMode) -> bool: @@ -197,19 +194,15 @@ def _need_to_upload(api: AliPCSApi, remotepath: str, check_name_mode: CheckNameM If `check_name_mode` is `refuse` and the `remotepath` exists, then it does not need to be uploaded. """ - try: - pcs_file = api.path(remotepath) - if pcs_file and check_name_mode == "refuse": - print(f"`{remotepath}` already exists.") - logger.debug("`_init_encrypt_io`: remote file already exists") - return False - return True - except Exception as err: - raise err + pcs_file = api.get_file(remotepath=remotepath) + if pcs_file is not None and check_name_mode == "refuse": + logger.debug("`_init_encrypt_io`: remote file already exists") + return False + return True def _init_encrypt_io( - localpath: str, + localpath: PathType, encrypt_password: bytes = b"", encrypt_type: EncryptType = EncryptType.No, ) -> Tuple[IO, int, int, int]: @@ -234,7 +227,7 @@ def _init_encrypt_io( def _rapid_upload( api: AliPCSApi, - localpath: str, + localpath: PathType, filename: str, dest_file_id: str, content_hash: str, @@ -261,333 +254,80 @@ def _rapid_upload( except AliPCSError as err: logger.warning("`_can_rapid_upload`: rapid_upload fails") - if err.error_code != 31079: # 31079: '未找到文件MD5，请使用上传API上传整个文件。' - remove_progress_task(task_id) + reset_progress_task(task_id) + if err.error_code == 31079: # 31079: '未找到文件MD5，请使用上传API上传整个文件。' + logger.debug("`_can_rapid_upload`: %s, no exist in remote", localpath) + return False + else: logger.warning("`_can_rapid_upload`: unknown error: %s", err) raise err - else: - reset_progress_task(task_id) - logger.debug("`_can_rapid_upload`: %s, no exist in remote", localpath) - - return False -def upload_one_by_one( +def upload_file( api: AliPCSApi, - from_to_list: List[FromTo], + from_to: FromTo[PathType, str], check_name_mode: CheckNameMode, - max_workers: int = CPU_NUM, encrypt_password: bytes = b"", encrypt_type: EncryptType = EncryptType.No, - slice_size: int = DEFAULT_SLICE_SIZE, - show_progress: bool = True, - user_id: Optional[str] = None, - user_name: Optional[str] = None, -): - """Upload files one by one with uploading the slices concurrently""" - - for from_to in from_to_list: - task_id = None - if show_progress: - task_id = _progress.add_task("upload", start=False, title=from_to.from_) - upload_file_concurrently( - api, - from_to, - check_name_mode, - max_workers=max_workers, - encrypt_password=encrypt_password, - encrypt_type=encrypt_type, - slice_size=slice_size, - task_id=task_id, - user_id=user_id, - user_name=user_name, - ) - - logger.debug("======== Uploading end ========") - - -@retry( - -1, - except_callback=lambda err, fail_count: logger.warning( - "`upload_file_concurrently`: fails: error: %s, fail_count: %s", - err, - fail_count, - exc_info=err, - ), -) -def upload_file_concurrently( - api: AliPCSApi, - from_to: FromTo, - check_name_mode: CheckNameMode, - max_workers: int = CPU_NUM, - encrypt_password: bytes = b"", - encrypt_type: EncryptType = EncryptType.No, - slice_size: int = DEFAULT_SLICE_SIZE, + slice_size: Union[str, int] = DEFAULT_SLICE_SIZE, + only_use_rapid_upload: bool = False, task_id: Optional[TaskID] = None, - user_id: Optional[str] = None, - user_name: Optional[str] = None, -): - """Uploading one file by uploading it's slices concurrently""" - - localpath, remotepath = from_to - - remotedir = posix_path_dirname(remotepath) - dest_dir = api.makedir_path(remotedir) - dest_file_id = dest_dir.file_id - - filename = posix_path_basename(remotepath) - - if not _need_to_upload(api, remotepath, check_name_mode): - remove_progress_task(task_id) - return - - info = _init_encrypt_io(localpath, encrypt_password=encrypt_password, encrypt_type=encrypt_type) - encrypt_io, encrypt_io_len, local_ctime, local_mtime = info - slice_size = adjust_slice_size(slice_size, encrypt_io_len) - part_number = math.ceil(encrypt_io_len / slice_size) - - # Progress bar - if task_id is not None and progress_task_exists(task_id): - _progress.update(task_id, total=encrypt_io_len) - _progress.start_task(task_id) - - slice_completed = 0 - slice_completeds = {} # current i-th index slice completed size - - def callback_for_slice(upload_url: str, monitor: MultipartEncoderMonitor): - if task_id is not None and progress_task_exists(task_id): - slice_completeds[upload_url] = monitor.bytes_read - current_compledted: int = sum(list(slice_completeds.values())) - _progress.update(task_id, completed=slice_completed + current_compledted) - - slice1k_hash = "" - content_hash = "" - - pcs_prepared_file = None - if encrypt_type == EncryptType.No and encrypt_io_len >= 1 * constant.OneK: - # Rapid Upload - - slice1k_bytes = encrypt_io.read(constant.OneK) - reset_encrypt_io(encrypt_io) - slice1k_hash = calc_sha1(slice1k_bytes) - - pcs_prepared_file = api.prepare_file( - filename, - dest_file_id, - encrypt_io_len, - slice1k_hash, - part_number=part_number, - check_name_mode=check_name_mode, - ) - if pcs_prepared_file.can_rapid_upload(): - content_hash = calc_sha1(encrypt_io) - proof_code = calc_proof_code(encrypt_io, encrypt_io_len, api.access_token) - - # Rapid upload - _rapid_upload( - api, - localpath, - filename, - dest_file_id, - content_hash, - proof_code, - encrypt_io_len, - check_name_mode=check_name_mode, - task_id=task_id, - ) - return - - try: - # Upload file slice - logger.debug("`upload_file_concurrently`: upload_slice starts") - - if not pcs_prepared_file: - pcs_prepared_file = api.create_file( - filename, - dest_file_id, - encrypt_io_len, - part_number=part_number, - check_name_mode=check_name_mode, - ) - - reset_encrypt_io(encrypt_io) - - def _upload_slice(item): - if not item: - return - - io, upload_url = item - - # Retry upload until success - retry( - -1, - except_callback=lambda err, fail_count: ( - io.seek(0, 0), - logger.warning( - "`upload_file_concurrently`: error: %s, fail_count: %s", - err, - fail_count, - exc_info=err, - ), - _wait_start(), - ), - )(api.upload_slice)( - io, - upload_url, - callback=functools.partial(callback_for_slice, upload_url), - ) - - slice_completeds.pop(upload_url) - - nonlocal slice_completed - slice_completed += total_len(io) - - semaphore = Semaphore(max_workers) - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futs = [] - i = 0 - for upload_url in pcs_prepared_file.upload_urls(): - semaphore.acquire() - - size = min(slice_size, encrypt_io_len - i) - if size == 0: - break - - data = encrypt_io.read(size) - io = BytesIO(data or b"") - - fut = executor.submit(sure_release, semaphore, _upload_slice, (io, upload_url)) - futs.append(fut) - - i += size - - as_completed(futs) - - file_id = pcs_prepared_file.file_id - upload_id = pcs_prepared_file.upload_id - assert file_id and upload_id - api.upload_complete(file_id, upload_id) - - remove_progress_task(task_id) - - logger.debug( - "`upload_file_concurrently`: upload_slice and combine_slices success, task_id: %s", - task_id, - ) - except Exception as err: - logger.warning("`upload_file_concurrently`: error: %s", err) - raise err - finally: - encrypt_io.close() - reset_progress_task(task_id) - - -def upload_many( - api: AliPCSApi, - from_to_list: List[FromTo], - check_name_mode: CheckNameMode = "overwrite", - encrypt_password: bytes = b"", - encrypt_type: EncryptType = EncryptType.No, - max_workers: int = CPU_NUM, - slice_size: int = DEFAULT_SLICE_SIZE, - show_progress: bool = True, - rapiduploadinfo_file: Optional[str] = None, - user_id: Optional[str] = None, - user_name: Optional[str] = None, -): - """Upload files concurrently that one file is with one connection""" - - excepts = {} - semaphore = Semaphore(max_workers) - with ThreadPoolExecutor(max_workers=max_workers) as executor: - futs = {} - for idx, from_to in enumerate(from_to_list): - semaphore.acquire() - task_id = None - if show_progress: - task_id = _progress.add_task("upload", start=False, title=from_to.from_) - - logger.debug("`upload_many`: Upload: index: %s, task_id: %s", idx, task_id) - - retry_upload_file = retry( - -1, - except_callback=lambda err, fail_count: logger.warning( - "`upload_file`: fails: error: %s, fail_count: %s", - err, - fail_count, - exc_info=err, - ), - )(upload_file) - - fut = executor.submit( - sure_release, - semaphore, - retry_upload_file, - api, - from_to, - check_name_mode, - encrypt_password=encrypt_password, - encrypt_type=encrypt_type, - slice_size=slice_size, - task_id=task_id, - user_id=user_id, - user_name=user_name, - ) - futs[fut] = from_to - - for fut in as_completed(futs): - e = fut.exception() - if e is not None: - from_to = futs[fut] - excepts[from_to] = e - - logger.debug("======== Uploading end ========") - - # Summary - if excepts: - table = Table(title="Upload Error", box=SIMPLE, show_edge=False) - table.add_column("From", justify="left", overflow="fold") - table.add_column("To", justify="left", overflow="fold") - table.add_column("Error", justify="left") - - for from_to, e in sorted(excepts.items()): - table.add_row(from_to.from_, Text(str(e), style="red")) + callback_for_monitor: Optional[Callable[[MultipartEncoderMonitor], None]] = None, +) -> None: + """Upload a file from `from_to[0]` to `from_to[1]` - _progress.console.print(table) + First try to rapid upload, if failed, then upload file's slices. + Raise exception if any error occurs. -def upload_file( - api: AliPCSApi, - from_to: FromTo, - check_name_mode: CheckNameMode, - encrypt_password: bytes = b"", - encrypt_type: EncryptType = EncryptType.No, - slice_size: int = DEFAULT_SLICE_SIZE, - task_id: Optional[TaskID] = None, - user_id: Optional[str] = None, - user_name: Optional[str] = None, - callback_for_monitor: Optional[Callable[[MultipartEncoderMonitor], None]] = None, -): - """Upload one file with one connection""" + Args: + api (AliPCSApi): AliPCSApi instance + from_to (FromTo[PathType, str]): FromTo instance decides the local path needed to upload and the remote path to upload to. + check_name_mode (CheckNameMode): CheckNameMode + encrypt_password (bytes, optional): Encrypt password. Defaults to b"". + encrypt_type (EncryptType, optional): Encrypt type. Defaults to EncryptType.No. + slice_size (Union[str, int], optional): Slice size. Defaults to DEFAULT_SLICE_SIZE. + only_use_rapid_upload (bool, optional): Only use rapid upload. If rapid upload fails, raise exception. Defaults to False. + task_id (Optional[TaskID], optional): Task ID. Defaults to None. + callback_for_monitor (Optional[Callable[[MultipartEncoderMonitor], None]], optional): Callback for progress monitor. Defaults to None. + + Examples: + - Upload one file to one remote directory + + ```python + >>> from alipcs_py.alipcs import AliPCSApi + >>> from alipcs_py.commands.upload import upload, from_tos + >>> api = AliPCSApi(...) + >>> remotedir = "/remote/dir" + >>> localpath = "/local/file" + >>> from_to = (localpath, remotedir) + >>> upload_file(api, from_to) + """ _wait_start() localpath, remotepath = from_to remotedir = posix_path_dirname(remotepath) - dest_dir = api.path(remotedir) + dest_dir = api.get_file(remotepath=remotedir) if dest_dir is None: - dest_dir = api.makedir_path(remotedir) + dest_dir = api.makedir_path(remotedir)[0] + else: + assert dest_dir.is_dir, f"`{remotedir}` is not a directory" dest_file_id = dest_dir.file_id filename = posix_path_basename(remotepath) if not _need_to_upload(api, remotepath, check_name_mode): + if task_id is not None: + print(f"`{remotepath}` already exists.") remove_progress_task(task_id) return info = _init_encrypt_io(localpath, encrypt_password=encrypt_password, encrypt_type=encrypt_type) encrypt_io, encrypt_io_len, local_ctime, local_mtime = info + if isinstance(slice_size, str): + slice_size = human_size_to_int(slice_size) slice_size = adjust_slice_size(slice_size, encrypt_io_len) part_number = math.ceil(encrypt_io_len / slice_size) @@ -602,45 +342,56 @@ def callback_for_slice(monitor: MultipartEncoderMonitor): if task_id is not None and progress_task_exists(task_id): _progress.update(task_id, completed=slice_completed + monitor.bytes_read) - slice1k_hash = "" - content_hash = "" - - pcs_prepared_file = None - if encrypt_type == EncryptType.No and encrypt_io_len >= 1 * constant.OneK: - # Rapid Upload - - slice1k_bytes = encrypt_io.read(constant.OneK) - reset_encrypt_io(encrypt_io) - slice1k_hash = calc_sha1(slice1k_bytes) - - pcs_prepared_file = api.prepare_file( - filename, - dest_file_id, - encrypt_io_len, - slice1k_hash, - part_number=part_number, - check_name_mode=check_name_mode, - ) - if pcs_prepared_file.can_rapid_upload(): - content_hash = calc_sha1(encrypt_io) - proof_code = calc_proof_code(encrypt_io, encrypt_io_len, api.access_token) - - # Rapid upload - _rapid_upload( - api, - localpath, + # Rapid Upload + try: + slice1k_hash = "" + content_hash = "" + pcs_prepared_file = None + if encrypt_type == EncryptType.No and encrypt_io_len >= 1 * constant.OneK: + slice1k_bytes = encrypt_io.read(constant.OneK) + reset_encrypt_io(encrypt_io) + slice1k_hash = calc_sha1(slice1k_bytes) + + pcs_prepared_file = api.prepare_file( filename, dest_file_id, - content_hash, - proof_code, encrypt_io_len, + slice1k_hash, + part_number=part_number, check_name_mode=check_name_mode, - task_id=task_id, ) - return + if pcs_prepared_file.can_rapid_upload(): + content_hash = calc_sha1(encrypt_io) + proof_code = calc_proof_code(encrypt_io, encrypt_io_len, api.access_token) + + # Rapid upload + ok = _rapid_upload( + api, + localpath, + filename, + dest_file_id, + content_hash, + proof_code, + encrypt_io_len, + check_name_mode=check_name_mode, + task_id=task_id, + ) + if ok: + return + except Exception as origin_err: + msg = f'Rapid upload "{localpath}" to "{remotepath}" failed. error: {origin_err}' + logger.debug(msg) + err = RapidUploadError(msg, localpath=localpath, remotepath=remotepath) + raise err from origin_err + + if only_use_rapid_upload: + msg = f'Only use rapid upload but rapid upload failed. localpath: "{localpath}", remotepath: "{remotepath}"' + logger.debug(msg) + err = RapidUploadError(msg, localpath=localpath, remotepath=remotepath) + raise err + # Upload file slice try: - # Upload file slice logger.debug("`upload_file`: upload_slice starts") if not pcs_prepared_file: @@ -696,14 +447,14 @@ def callback_for_slice(monitor: MultipartEncoderMonitor): ) slice_idx += 1 break - except Exception as err: + except Exception as origin_err: fail_count += 1 io.seek(0, 0) - logger.warning( - "`upload_file`: `upload_slice`: error: %s, fail_count: %s", - err, + logger.debug( + "Upload slice failed. error: %s, fail_count: %s", + origin_err, fail_count, - exc_info=err, + exc_info=origin_err, ) _wait_start() @@ -729,9 +480,11 @@ def callback_for_slice(monitor: MultipartEncoderMonitor): "`upload_file`: upload_slice and combine_slices success, task_id: %s", task_id, ) - except Exception as err: - logger.warning("`upload_file`: error: %s", err) - raise err + except Exception as origin_err: + msg = f'Upload "{localpath}" to "{remotepath}" failed. error: {origin_err}' + logger.debug(msg) + err = UploadError(msg, localpath=localpath, remotepath=remotepath) + raise err from origin_err finally: encrypt_io.close() reset_progress_task(task_id) From b91f2d057407691994905d063bd52530fdd94ded Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 20:08:48 +0800 Subject: [PATCH 17/32] Use new apis and clean code --- alipcs_py/commands/display.py | 2 +- alipcs_py/commands/file_operators.py | 45 ++++++++++---- alipcs_py/commands/list_files.py | 20 +++---- alipcs_py/commands/play.py | 28 ++++----- alipcs_py/commands/server.py | 14 ++--- alipcs_py/commands/share.py | 90 +++++++++++++++++----------- alipcs_py/commands/sync.py | 67 ++++++++++----------- 7 files changed, 146 insertions(+), 120 deletions(-) diff --git a/alipcs_py/commands/display.py b/alipcs_py/commands/display.py index 5142282..28461cc 100644 --- a/alipcs_py/commands/display.py +++ b/alipcs_py/commands/display.py @@ -179,7 +179,7 @@ def display_from_to(*from_to_list: FromTo): table.add_column("To", justify="left", overflow="fold") for from_to in from_to_list: - table.add_row(from_to.from_, from_to.to_) + table.add_row(*from_to) console = Console() console.print(table) diff --git a/alipcs_py/commands/file_operators.py b/alipcs_py/commands/file_operators.py index 67277ac..671ed91 100644 --- a/alipcs_py/commands/file_operators.py +++ b/alipcs_py/commands/file_operators.py @@ -1,9 +1,8 @@ -from typing import Optional, List +from typing import List -from alipcs_py.alipcs import AliPCSApi, AliPCSError +from alipcs_py.alipcs import AliPCSApi from alipcs_py.alipcs.inner import PcsFile, FromTo from alipcs_py.common.path import ( - posix_path_dirname, split_posix_path, posix_path_basename, join_path, @@ -12,9 +11,11 @@ def makedir(api: AliPCSApi, *remotedirs: str, show: bool = False) -> List[PcsFile]: + """Make directories (`remotepaths`).""" + pcs_files = [] for d in remotedirs: - pf = api.makedir_path(d) + pf = api.makedir_path(d)[0] pcs_files.append(pf) if show: @@ -24,6 +25,11 @@ def makedir(api: AliPCSApi, *remotedirs: str, show: bool = False) -> List[PcsFil def move(api: AliPCSApi, *remotepaths: str, show: bool = False): + """Move files or directories to a destination directory. + + Move the paths(`remotepaths[:-1]`) to the directory(`remotepaths[-1]`). + """ + assert len(remotepaths) > 1 dest_remotepath = remotepaths[-1] @@ -31,22 +37,22 @@ def move(api: AliPCSApi, *remotepaths: str, show: bool = False): # Make sure destination directory exists if not pcs_files[-1]: - dest_pcs_file = api.makedir_path(dest_remotepath) + dest_pcs_file = api.makedir_path(dest_remotepath)[0] pcs_files[-1] = dest_pcs_file file_ids = [pf.file_id for pf in pcs_files if pf] oks = api.move(*file_ids) - from_to_list = [] + from_to_list: List[FromTo[str, str]] = [] j = 0 for i, pf in enumerate(pcs_files[:-1]): if not pf: continue if oks[j]: - from_to = FromTo( - from_=remotepaths[i], - to_=join_path(dest_remotepath, posix_path_basename(remotepaths[i])), + from_to = ( + remotepaths[i], + join_path(dest_remotepath, posix_path_basename(remotepaths[i])), ) from_to_list.append(from_to) j += 1 @@ -56,6 +62,13 @@ def move(api: AliPCSApi, *remotepaths: str, show: bool = False): def rename(api: AliPCSApi, remotepath: str, new_name: str, show: bool = False): + """Rename a file or directory. + + e.g. + rename(api, remotepath="/some/file", new_name="new_file") is equal to + move(api, "/some/file", "/some/new_file") + """ + pcs_file = api.path(remotepath) if not pcs_file: @@ -69,25 +82,33 @@ def rename(api: AliPCSApi, remotepath: str, new_name: str, show: bool = False): def copy(api: AliPCSApi, *remotepaths: str, show: bool = False): + """Copy files or directories to a destination directory. + + Copy the paths(`remotepaths[:-1]`) to the directory(`remotepaths[-1]`). + """ + assert len(remotepaths) > 1 dest_remotepath = remotepaths[-1] pcs_files = api.paths(*remotepaths) + non_exist = [(rp, pf) for rp, pf in zip(remotepaths[:-1], pcs_files[:-1]) if pf is None] + assert len(non_exist) == 0, f"Paths not exist: {non_exist}" + # Make sure destination directory exists if not pcs_files[-1]: - dest_pcs_file = api.makedir_path(dest_remotepath) + dest_pcs_file = api.makedir_path(dest_remotepath)[0] pcs_files[-1] = dest_pcs_file file_ids = [pf.file_id for pf in pcs_files if pf] - new_pfs = api.copy(*file_ids) - if show: display_files(new_pfs, "", show_file_id=True, show_absolute_path=True) def remove(api: AliPCSApi, *remotepaths: str): + """Remove files or directories to the trash.""" + pcs_files = api.paths(*remotepaths) file_ids = [pf.file_id for pf in pcs_files if pf] api.remove(*file_ids) diff --git a/alipcs_py/commands/list_files.py b/alipcs_py/commands/list_files.py index f679f5d..2a58a25 100644 --- a/alipcs_py/commands/list_files.py +++ b/alipcs_py/commands/list_files.py @@ -41,19 +41,15 @@ def list_file( csv: bool = False, only_dl_link: bool = False, ): - pcs_file: Optional[PcsFile] - if file_id: - pcs_file = api.meta(file_id, share_id=share_id)[0] - else: - pcs_file = api.path(remotepath, share_id=share_id) - if not pcs_file: + pcs_file = api.get_file(remotepath=remotepath, file_id=file_id, share_id=share_id) + if pcs_file is None: return is_dir = pcs_file.is_dir if is_dir: - pcs_files = api.list_path( - remotepath, - file_id=pcs_file.file_id, + pcs_files = [] + for sub_pf in api.list_iter( + pcs_file.file_id, share_id=share_id, desc=desc, name=name, @@ -62,7 +58,9 @@ def list_file( all=all, limit=limit, url_expire_sec=url_expire_sec, - ) + ): + sub_pf.path = join_path(remotepath, sub_pf.path) + pcs_files.append(sub_pf) else: pcs_files = [pcs_file] @@ -95,7 +93,7 @@ def list_file( if pcs_file.is_dir: list_file( api, - join_path(remotepath, pcs_file.name), + pcs_file.path, file_id=pcs_file.file_id, share_id=share_id, desc=desc, diff --git a/alipcs_py/commands/play.py b/alipcs_py/commands/play.py index 5ab5734..c426531 100644 --- a/alipcs_py/commands/play.py +++ b/alipcs_py/commands/play.py @@ -1,12 +1,10 @@ -from typing import Optional, List, Dict +from typing import Optional, List from enum import Enum -from pathlib import Path import os import shutil import subprocess import random import time -from urllib.parse import quote from alipcs_py.alipcs import AliPCSApi, PcsFile from alipcs_py.alipcs.errors import AliPCSError @@ -14,7 +12,6 @@ from alipcs_py.commands.download import USER_AGENT from alipcs_py.commands.errors import CommandError from alipcs_py.common.file_type import MEDIA_EXTS -from alipcs_py.common.path import join_path from rich import print @@ -126,7 +123,7 @@ def play_file( shared_pcs_filename = pcs_file.name use_local_server = False remote_temp_dir = "/__alipcs_py_temp__" - pcs_temp_dir = api.path(remote_temp_dir) or api.makedir_path(remote_temp_dir) + pcs_temp_dir = api.path(remote_temp_dir) or api.makedir_path(remote_temp_dir)[0] pf = api.transfer_shared_files([shared_pcs_file_id], pcs_temp_dir.file_id, share_id)[0] target_file_id = pf.file_id while True: @@ -189,21 +186,18 @@ def play_dir( out_cmd: bool = False, local_server: str = "", ): - if pcs_file.path.startswith("/"): - remotefiles = list(api.list_path(pcs_file.path, share_id=share_id)) - else: - remotefiles = list(api.list_iter(pcs_file.file_id, share_id=share_id)) - remotefiles = sift(remotefiles, sifters, recursive=recursive) + sub_pcs_files = list(api.list_iter(pcs_file.file_id, share_id=share_id)) + sub_pcs_files = sift(sub_pcs_files, sifters, recursive=recursive) if shuffle: rg = random.Random(time.time()) - rg.shuffle(remotefiles) + rg.shuffle(sub_pcs_files) - for rp in remotefiles[from_index:]: - if rp.is_file: + for pf in sub_pcs_files[from_index:]: + if pf.is_file: play_file( api, - rp, + pf, share_id=share_id, player=player, player_params=player_params, @@ -216,7 +210,7 @@ def play_dir( if recursive: play_dir( api, - rp, + pf, share_id=share_id, sifters=sifters, recursive=recursive, @@ -293,8 +287,8 @@ def play( ) for file_id in file_ids: - rpf = api.meta(file_id, share_id=share_id)[0] - if not rpf: + rpf = api.get_file(file_id=file_id, share_id=share_id) + if rpf is None: print(f"[yellow]WARNING[/yellow]: file_id `{file_id}` does not exist.") continue diff --git a/alipcs_py/commands/server.py b/alipcs_py/commands/server.py index 2db5768..543fd3a 100644 --- a/alipcs_py/commands/server.py +++ b/alipcs_py/commands/server.py @@ -128,18 +128,18 @@ async def handle_request( if is_dir: chunks = ["/"] + (remotepath.split("/") if remotepath != "" else []) navigation = [(i - 1, "../" * (len(chunks) - i), name) for i, name in enumerate(chunks, 1)] - pcs_files = _api.list_path_iter(_rp, desc=desc, name=name, time=time, size=size) entries = [] - for f in pcs_files: - p = Path(f.path) + pcs_files = _api.list_iter(rpf.file_id, desc=desc, name=name, time=time, size=size) + for pf in pcs_files: + p = rpf.path / Path(pf.path) entries.append( ( - f.file_id, - f.is_dir, + pf.file_id, + pf.is_dir, p.name, quote(p.name), - f.size, - format_date(f.updated_at or 0), + pf.size, + format_date(pf.updated_at or 0), ) ) cn = _html_tempt.render(root_dir=remotepath, navigation=navigation, entries=entries) diff --git a/alipcs_py/commands/share.py b/alipcs_py/commands/share.py index 79ae490..dd84153 100644 --- a/alipcs_py/commands/share.py +++ b/alipcs_py/commands/share.py @@ -1,4 +1,4 @@ -from typing import List, Dict, Set +from typing import List, Dict, Set, Union from pathlib import PurePosixPath from collections import deque import re @@ -13,11 +13,11 @@ display_shared_links, ) from alipcs_py.commands.download import ( + DEFAULT_CHUNK_SIZE, + DEFAULT_CONCURRENCY, download, Downloader, DEFAULT_DOWNLOADER, - DownloadParams, - DEFAULT_DOWNLOADPARAMS, ) from alipcs_py.commands.play import play, Player, DEFAULT_PLAYER @@ -83,55 +83,57 @@ def save_shared_by_file_ids( file_ids: List[str], password: str = "", ): + """Save shared files to the remote directory `remotedir`. Ignore existed files.""" + assert share_id api.get_share_token(share_id, share_password=password) file_ids = file_ids or ["root"] - sfs = api.meta(*file_ids, share_id=share_id) - for sf in sfs: - if not sf.path: - sf.path = sf.name - shared_files = deque(sfs) + shared_pcs_files = deque() + for file_id in file_ids: + pf = api.get_file(file_id=file_id, share_id=share_id) + if pf is not None: + shared_pcs_files.append(pf) - # Record the remotedir of each shared_file - _remotedirs: Dict[str, str] = {} - for sp in shared_files: - _remotedirs[sp.file_id] = remotedir + # Record the remote directory of each shared_file + shared_file_id_to_remotedir: Dict[str, str] = {} + for sp in shared_pcs_files: + shared_file_id_to_remotedir[sp.file_id] = remotedir - # Map the remotedir to its pcs_file - dest_pcs_files: Dict[str, PcsFile] = {} + # Map the remote directory to its pcs_file + remotedir_to_its_pcs_file: Dict[str, PcsFile] = {} - while shared_files: - shared_file = shared_files.popleft() - rd = _remotedirs[shared_file.file_id] + while shared_pcs_files: + shared_file = shared_pcs_files.popleft() + remote_dir = shared_file_id_to_remotedir[shared_file.file_id] - # Make sure remote dir exists - if rd not in dest_pcs_files: - dest_pcs_files[rd] = api.makedir_path(rd) - dest_pcs_file = dest_pcs_files[rd] + # Make sure remote directory exists + if remote_dir not in remotedir_to_its_pcs_file: + remotedir_to_its_pcs_file[remote_dir] = api.makedir_path(remote_dir)[0] + dest_pcs_file = remotedir_to_its_pcs_file[remote_dir] - if not shared_file.is_root() and not remotepath_exists(api, shared_file.name, rd): + if not shared_file.is_root() and not remotepath_exists(api, shared_file.name, dest_pcs_file.file_id): api.transfer_shared_files( [shared_file.file_id], dest_pcs_file.file_id, share_id, auto_rename=False, ) - print(f"save: `{shared_file.path}` to `{rd}`") + print(f"save: `{shared_file.path}` to `{remote_dir}`") else: # Ignore existed file if shared_file.is_file: - print(f"[yellow]WARNING[/]: `{shared_file.path}` has be in `{rd}`") + print(f"[yellow]WARNING[/]: `{shared_file.path}` has be in `{remote_dir}`") continue else: # shared_file.is_dir - sub_files = list(api.list_path_iter(shared_file.path, file_id=shared_file.file_id, share_id=share_id)) + sub_files = list(api.list_iter(shared_file.file_id, share_id=share_id)) - rd = (PurePosixPath(rd) / shared_file.name).as_posix() + remote_dir = (PurePosixPath(remote_dir) / shared_file.name).as_posix() for sp in sub_files: - _remotedirs[sp.file_id] = rd - shared_files.extendleft(sub_files[::-1]) + shared_file_id_to_remotedir[sp.file_id] = remote_dir + shared_pcs_files.extendleft(sub_files[::-1]) def save_shared( @@ -142,6 +144,8 @@ def save_shared( file_ids: List[str] = [], password: str = "", ): + """Save shared files of the shared url to the remote directory `remotedir`.""" + assert remotedir.startswith("/"), "`remotedir` must be an absolute path" assert int(bool(share_id)) ^ int(bool(share_url)), "`share_id` and `share_url` only can be given one" @@ -175,6 +179,8 @@ def list_shared_files( show_absolute_path: bool = False, csv: bool = False, ): + """List shared files in the shared url or shared id.""" + assert int(bool(share_id)) ^ int(bool(share_url)), "`share_id` and `share_url` only can be given one" share_url = _redirect(share_url) @@ -215,17 +221,19 @@ def list_shared_files( ) -def remotepath_exists(api: AliPCSApi, name: str, rd: str, _cache: Dict[str, Set[str]] = {}) -> bool: - names = _cache.get(rd) +def remotepath_exists(api: AliPCSApi, name: str, remote_file_id: str, _cache: Dict[str, Set[str]] = {}) -> bool: + """Check if the `name` exists in the remote directory `remote_file_id`.""" + + names = _cache.get(remote_file_id) if not names: - names = set([sp.name for sp in api.list_path_iter(rd)]) - _cache[rd] = names + names = set(sp.name for sp in api.list_iter(remote_file_id)) + _cache[remote_file_id] = names return name in names def download_shared( api: AliPCSApi, - remotepaths: List[str], + remotepaths: List[Union[str, PcsFile]], file_ids: List[str], localdir: str, share_id: str = "", @@ -235,10 +243,15 @@ def download_shared( recursive: bool = False, from_index: int = 0, downloader: Downloader = DEFAULT_DOWNLOADER, - downloadparams: DownloadParams = DEFAULT_DOWNLOADPARAMS, + concurrency: int = DEFAULT_CONCURRENCY, + chunk_size: Union[str, int] = DEFAULT_CHUNK_SIZE, + show_progress: bool = False, + max_retries: int = 2, out_cmd: bool = False, encrypt_password: bytes = b"", ): + """Download shared files in the shared url or shared id.""" + assert int(bool(share_id)) ^ int(bool(share_url)), "`share_id` and `share_url` only can be given one" share_url = _redirect(share_url) @@ -266,7 +279,10 @@ def download_shared( recursive=recursive, from_index=from_index, downloader=downloader, - downloadparams=downloadparams, + concurrency=concurrency, + chunk_size=chunk_size, + show_progress=show_progress, + max_retries=max_retries, out_cmd=out_cmd, encrypt_password=encrypt_password, ) @@ -290,6 +306,8 @@ def play_shared( out_cmd: bool = False, local_server: str = "", ): + """Play shared files in the shared url or shared id.""" + assert int(bool(share_id)) ^ int(bool(share_url)), "`share_id` and `share_url` only can be given one" share_url = _redirect(share_url) @@ -326,6 +344,8 @@ def play_shared( def get_share_token(api: AliPCSApi, share_id: str, share_url: str = "", password: str = "") -> str: + """Initiate a shared link (or id) and get the share token.""" + assert int(bool(share_id)) ^ int(bool(share_url)), "`share_id` and `share_url` only can be given one" share_url = _redirect(share_url) diff --git a/alipcs_py/commands/sync.py b/alipcs_py/commands/sync.py index fa78310..78f90e8 100644 --- a/alipcs_py/commands/sync.py +++ b/alipcs_py/commands/sync.py @@ -1,12 +1,13 @@ -from typing import Optional, List, Tuple +from typing import List, Tuple from pathlib import Path +import os from alipcs_py.alipcs import AliPCSApi, PcsFile, FromTo -from alipcs_py.common.path import walk, join_path +from alipcs_py.common.path import PathType, join_path from alipcs_py.common.crypto import calc_sha1 from alipcs_py.common.constant import CPU_NUM from alipcs_py.common.io import EncryptType -from alipcs_py.commands.upload import UploadType, upload as _upload, DEFAULT_SLICE_SIZE +from alipcs_py.commands.upload import upload, DEFAULT_SLICE_SIZE from alipcs_py.commands.log import get_logger from rich import print @@ -23,69 +24,61 @@ def sync( max_workers: int = CPU_NUM, slice_size: int = DEFAULT_SLICE_SIZE, show_progress: bool = True, - rapiduploadinfo_file: Optional[str] = None, - user_id: Optional[str] = None, - user_name: Optional[str] = None, ): + """Sync local directory to remote directory.""" + localdir = Path(localdir).as_posix() remotedir = Path(remotedir).as_posix() - rpf = api.makedir_path(remotedir) - assert rpf and rpf.is_dir, "remotedir must be a directory" - - if remotedir == "/": - remotedir_len = 0 - else: - remotedir_len = len(remotedir) + remote_pcs_file = api.makedir_path(remotedir)[0] + assert remote_pcs_file and remote_pcs_file.is_dir, "remotedir must be a directory" - all_pcs_files = { - pcs_file.path[remotedir_len + 1 :]: pcs_file - for pcs_file in api.list_path_iter(remotedir, recursive=True, include_dir=False) + sub_path_to_its_pcs_file = { + pcs_file.path: pcs_file + for pcs_file in api.list_iter(remote_pcs_file.file_id, recursive=True, include_dir=False) } - fts: List[FromTo] = [] - check_list: List[Tuple[str, PcsFile]] = [] + needed_uploads: List[FromTo[PathType, str]] = [] + needed_checks: List[Tuple[Path, PcsFile]] = [] all_localpaths = set() - for localpath in walk(localdir): - path = localpath[len(localdir) + 1 :] - all_localpaths.add(path) + for root, _, filenames in os.walk(localdir): + for filename in filenames: + localpath = Path(root[len(localdir) + 1 :]) / filename + localpath_posix = localpath.as_posix() + all_localpaths.add(localpath_posix) - if path not in all_pcs_files: - fts.append(FromTo(localpath, join_path(remotedir, path))) - else: - check_list.append((localpath, all_pcs_files[path])) + if localpath_posix not in sub_path_to_its_pcs_file: + needed_uploads.append((root / localpath, join_path(remotedir, localpath))) + else: + needed_checks.append((root / localpath, sub_path_to_its_pcs_file[localpath_posix])) - for lp, pf in check_list: - sha1 = calc_sha1(Path(lp).open("rb")) + for lp, pf in needed_checks: + sha1 = calc_sha1(lp.open("rb")) if pf.rapid_upload_info and sha1.lower() != pf.rapid_upload_info.content_hash.lower(): - fts.append(FromTo(lp, pf.path)) + needed_uploads.append((lp, pf.path)) need_deleted_file_ids = [] - for rp in all_pcs_files.keys(): + for rp in sub_path_to_its_pcs_file.keys(): if rp not in all_localpaths: - need_deleted_file_ids.append(all_pcs_files[rp].file_id) + need_deleted_file_ids.append(sub_path_to_its_pcs_file[rp].file_id) logger.debug( "`sync`: all localpaths: %s, localpaths needed to upload: %s, remotepaths needed to delete: %s", len(all_localpaths), - len(fts), + len(needed_uploads), len(need_deleted_file_ids), ) - _upload( + upload( api, - fts, - upload_type=UploadType.Many, + needed_uploads, check_name_mode="overwrite", encrypt_password=encrypt_password, encrypt_type=encrypt_type, max_workers=max_workers, slice_size=slice_size, show_progress=show_progress, - rapiduploadinfo_file=rapiduploadinfo_file, - user_id=user_id, - user_name=user_name, ) if need_deleted_file_ids: From 036c40b54cf6c7852f2517952260b7fac24b55cf Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 20:09:49 +0800 Subject: [PATCH 18/32] Add comments --- alipcs_py/common/downloader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/alipcs_py/common/downloader.py b/alipcs_py/common/downloader.py index f06f02c..1001f6a 100644 --- a/alipcs_py/common/downloader.py +++ b/alipcs_py/common/downloader.py @@ -10,6 +10,8 @@ class MeDownloader: + """Download the content from `range_request_io` to `localpath`""" + def __init__( self, range_request_io: RangeRequestIO, @@ -44,8 +46,7 @@ def _init_fd(self): self.fd = fd def download(self): - """ - Download the url content to `localpath` + """Download the url content to `localpath` Args: continue_ (bool): If set to True, only downloading the remain content depended on From 1fc7f69637ad9b3f6cb93b9ee1a94373a4b4b971 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 20:10:10 +0800 Subject: [PATCH 19/32] Ignore existed directories when creating them --- alipcs_py/app/account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alipcs_py/app/account.py b/alipcs_py/app/account.py index 52c4457..f789f78 100644 --- a/alipcs_py/app/account.py +++ b/alipcs_py/app/account.py @@ -259,7 +259,7 @@ def save(self, data_path: Optional[PathType] = None): data_path = Path(data_path).expanduser() if not data_path.parent.exists(): - data_path.parent.mkdir(parents=True) + data_path.parent.mkdir(parents=True, exist_ok=True) apis = self._apis self._apis = {} # Ignore to save apis From 4eb720c01a3799b276bbfbf6d3a136d821965999 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 22:41:43 +0800 Subject: [PATCH 20/32] Update api --- alipcs_py/storage/store.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/alipcs_py/storage/store.py b/alipcs_py/storage/store.py index eecbd4d..40466b2 100644 --- a/alipcs_py/storage/store.py +++ b/alipcs_py/storage/store.py @@ -146,18 +146,17 @@ def get_share_token(self, share_id: str, share_password: str = "") -> str: return token - def meta(self, *args, **kwargs): - share_id = kwargs.get("share_id") - - pcs_files = super().meta(*args, **kwargs) + def meta(self, file_id: str, share_id: Optional[str] = None) -> Optional[PcsFile]: + pcs_file = super().meta(file_id, share_id=share_id) + if pcs_file is None: + return None if not self._sharedstore: - return pcs_files + return pcs_file if share_id: - for pcs_file in pcs_files: - self._sharedstore.add_shared_file(share_id, pcs_file) + self._sharedstore.add_shared_file(share_id, pcs_file) - return pcs_files + return pcs_file def list(self, *args, **kwargs): share_id = kwargs.get("share_id") From f476f3a0fc5be1d72f398ffd54fcdf6a582eda51 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 22:44:02 +0800 Subject: [PATCH 21/32] Update api --- alipcs_py/app/app.py | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/alipcs_py/app/app.py b/alipcs_py/app/app.py index 2a6bfd2..730e5bf 100644 --- a/alipcs_py/app/app.py +++ b/alipcs_py/app/app.py @@ -47,13 +47,12 @@ from alipcs_py.commands.download import ( download as _download, Downloader, - DownloadParams, DEFAULT_DOWNLOADER, DEFAULT_CONCURRENCY, DEFAULT_CHUNK_SIZE, ) from alipcs_py.commands.play import play as _play, Player, DEFAULT_PLAYER -from alipcs_py.commands.upload import upload as _upload, from_tos, CPU_NUM, UploadType +from alipcs_py.commands.upload import upload as _upload, from_tos, CPU_NUM from alipcs_py.commands.sync import sync as _sync from alipcs_py.commands import share as _share from alipcs_py.commands.server import start_server @@ -328,7 +327,7 @@ def get_command(self, ctx, cmd_name): return click.Group.get_command(self, ctx, normal_cmd_name) def list_commands(self, ctx): - return self.commands + return list(self.commands) _APP_DOC = f"""AliPCS App v{__version__} @@ -1047,7 +1046,9 @@ def download( recursive=recursive, from_index=from_index, downloader=getattr(Downloader, downloader), - downloadparams=DownloadParams(concurrency=concurrency, chunk_size=chunk_size, quiet=quiet), + concurrency=concurrency, + chunk_size=chunk_size, + show_progress=not quiet, out_cmd=out_cmd, encrypt_password=encrypt_password, ) @@ -1063,7 +1064,9 @@ def download( recursive=recursive, from_index=from_index, downloader=getattr(Downloader, downloader), - downloadparams=DownloadParams(concurrency=concurrency, chunk_size=chunk_size, quiet=quiet), + concurrency=concurrency, + chunk_size=chunk_size, + show_progress=not quiet, out_cmd=out_cmd, encrypt_password=encrypt_password, ) @@ -1142,6 +1145,7 @@ def play( sifters.append(ExcludeSifter(exclude_regex, regex=True)) local_server = "" + ps = None if use_local_server: if share_id or file_id: assert ValueError("Recently local server can't play others shared items and using `file_id`") @@ -1210,24 +1214,13 @@ def play( local_server=local_server, ) - if use_local_server: + if use_local_server and ps is not None: ps.terminate() @app.command() @click.argument("localpaths", nargs=-1, type=str) @click.argument("remotedir", nargs=1, type=str) -@click.option( - "--upload-type", - "-t", - type=click.Choice([t.name for t in UploadType]), - default=UploadType.Many.name, - help=( - "上传方式，Many: 同时上传多个文件，" - "One: 一次只上传一个文件，但同时上传文件的多个分片 " - "(阿里网盘不支持单文件并发上传。`upload --upload-type One` 失效)" - ), -) @click.option("--encrypt-password", "--ep", type=str, default=None, help="加密密码，默认使用用户设置的") @click.option( "--encrypt-type", @@ -1247,7 +1240,6 @@ def upload( ctx, localpaths, remotedir, - upload_type, encrypt_password, encrypt_type, max_workers, @@ -1256,9 +1248,6 @@ def upload( ): """上传文件""" - if upload_type == UploadType.One.name: - raise ValueError("阿里网盘不支持单文件并发上传。`upload --upload-type One` 失效") - # Keyboard listener start keyboard_listener_start() @@ -1283,7 +1272,6 @@ def upload( _upload( api, from_to_list, - upload_type=getattr(UploadType, upload_type), check_name_mode=check_name_mode, encrypt_password=encrypt_password, encrypt_type=getattr(EncryptType, encrypt_type), @@ -1637,8 +1625,8 @@ def cleanstore(ctx): print("App configuration `[share] store is false`. So the command does not work") return - api: AliPCSApiMixWithSharedStore = _recent_api(ctx) - if not api: + api = _recent_api(ctx) + if not isinstance(api, AliPCSApiMixWithSharedStore): return store = api.sharedstore From c2cea9ca5e3892d2861aab981072be9a1cd9be4f Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 22:46:04 +0800 Subject: [PATCH 22/32] Add pytest-cov and Faker for testing --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 8a25929..add8366 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,8 @@ passlib = ">=1.7" [tool.poetry.group.dev.dependencies] pytest = ">=7.4" +pytest-cov = ">=5.0" +Faker = ">=24" ruff = ">=0.3" setuptools = ">=69.0" cython = ">=3.0" From 646509f7d2da72ff84143a372cc49a16101b2d03 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 22:46:58 +0800 Subject: [PATCH 23/32] Ignore local script --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index f252209..0108a9d 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ docs/_build/ /bin.py /alipcs_py/common/simple_cipher.c /alipcs_py/common/simple_cipher.html + +# Secret things +/run-tests.sh From 13654464b2ece865b468f3bc7af8af3dafb5f6a4 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 22:47:53 +0800 Subject: [PATCH 24/32] Only test on 3.9 and 3.12 --- .github/workflows/{main.yml => tests.yml} | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) rename .github/workflows/{main.yml => tests.yml} (80%) diff --git a/.github/workflows/main.yml b/.github/workflows/tests.yml similarity index 80% rename from .github/workflows/main.yml rename to .github/workflows/tests.yml index 0433760..3a8a26c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,13 @@ name: AliPCS-Py Build & Test -on: [push, pull_request] +on: + push: + branches: + - master + pull_request: + branches: + - master + - release/* jobs: build-test: @@ -8,14 +15,14 @@ jobs: strategy: matrix: os: [windows-latest, ubuntu-latest] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ["3.8", "3.12"] defaults: run: shell: bash steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} architecture: x64 @@ -35,7 +42,7 @@ jobs: - name: Test with pytest run: | poetry run python build.py build_ext --inplace - poetry run pytest -s tests/test_common.py + poetry run pytest -v -s - name: Test package run: | poetry build -f sdist From b09ee98e93b387b3be2a330332efb65ac3ce8c90 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 22:55:59 +0800 Subject: [PATCH 25/32] Add add test cases --- tests/conftest.py | 79 ++++++ tests/datas.py | 15 ++ tests/test-datas/demo-directory.tar.gz | Bin 0 -> 4795 bytes tests/test_alipcs.py | 226 ++++++++++++++++ tests/test_alipcsapi.py | 345 +++++++++++++++++++++++++ tests/test_commands.py | 332 ++++++++++++++++++++++++ tests/test_common.py | 22 -- 7 files changed, 997 insertions(+), 22 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/datas.py create mode 100644 tests/test-datas/demo-directory.tar.gz create mode 100644 tests/test_alipcs.py create mode 100644 tests/test_alipcsapi.py create mode 100644 tests/test_commands.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..427e385 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,79 @@ +import os +from pathlib import Path + +from alipcs_py import AliPCS, AliPCSApi +from alipcs_py.commands.upload import upload, from_tos +from alipcs_py.common.platform import IS_WIN + +from tests.datas import REFRESH_TOKEN, Datas + +import pytest + + +TEST_ROOT = "/AliPCS-Py-test" +LOCAL_DIR = Path("tests", "test-datas", "demo-directory") +TEST_DATA_PATH = LOCAL_DIR.parent / "demo-directory.tar.gz" + + +@pytest.fixture(scope="session") +def uncompress_test_data(): + if LOCAL_DIR.exists(): + if IS_WIN: + os.system(f"rd /s /q {LOCAL_DIR}") + else: + os.system(f"rm -rf {LOCAL_DIR}") + + assert TEST_DATA_PATH.exists() + if IS_WIN: + os.system(f"tar -xf {TEST_DATA_PATH} -C tests\\test-datas") + else: + os.system(f"tar -xf {TEST_DATA_PATH} -C tests/test-datas") + + yield + + if LOCAL_DIR.exists(): + if IS_WIN: + os.system(f"rd /s /q {LOCAL_DIR}") + else: + os.system(f"rm -rf {LOCAL_DIR}") + + +@pytest.fixture(scope="session") +def alipcsapi(uncompress_test_data) -> AliPCSApi: + return AliPCSApi(refresh_token=REFRESH_TOKEN) + + +@pytest.fixture(scope="session") +def alipcs(alipcsapi: AliPCSApi) -> AliPCS: + return alipcsapi._alipcs + + +@pytest.fixture(scope="session") +def datas(alipcsapi: AliPCSApi): + if REFRESH_TOKEN == "": + return + + local_paths = [] + local_dir = LOCAL_DIR + for root, _, files in os.walk(local_dir): + for fl in files: + local_paths.append(str(Path(root, fl))) + + remote_dir = TEST_ROOT + "/-------" + remote_dir_pcs_file = alipcsapi.makedir_path(remote_dir)[0] + from_paths = [str(local_dir / fn) for fn in os.listdir(local_dir)] + from_to_list = from_tos(from_paths, remote_dir) + + upload(alipcsapi, from_to_list) + + yield Datas( + local_dir=str(local_dir), + local_paths=local_paths, + remote_dir=remote_dir, + remote_dir_pcs_file=remote_dir_pcs_file, + remote_paths=[to_ for _, to_ in from_to_list], + ) + + pf = alipcsapi.meta_by_path(TEST_ROOT) + assert pf is not None + alipcsapi.remove(pf.file_id) diff --git a/tests/datas.py b/tests/datas.py new file mode 100644 index 0000000..fe1f241 --- /dev/null +++ b/tests/datas.py @@ -0,0 +1,15 @@ +import os +from dataclasses import dataclass + +from alipcs_py.alipcs.inner import PcsFile + +REFRESH_TOKEN = os.getenv("REFRESH_TOKEN", "") + + +@dataclass +class Datas: + local_dir: str + local_paths: list[str] + remote_dir: str + remote_dir_pcs_file: PcsFile + remote_paths: list[str] diff --git a/tests/test-datas/demo-directory.tar.gz b/tests/test-datas/demo-directory.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..1b39b81c26e32a03932242a4d99344530288635e GIT binary patch literal 4795 zcmV;s5=8AEiwFP!000001MOYQcH2f0^;%!iKQI_R#kFk5an7+PKCyGWZUPOUEdY&l zH%Mc?e(F|(ltPU~nv$S6>Dox*3lxA`@4A(#+WafiR;Aw9{@*X2LM9}uy7rkF!VC%wP}*){;#5(DhK%AT1~e@ zVzTxTO~Lo;|K)VHi28pF`S$hSmd2^1ZT;7W9^;t)pUvk7*MBv=TtxjphFld&xYk0C zD>@VFO1n~&N?I@I&pa1x**l%G>QA*IjqkjubgqPIlqGXvmG5jLdee!t^TKTC_PO|} z`C@vpHDo<3nFZJcA+uWhMEs^?TGAmESsb!8s&=Z{C?~vh*P^wiu(B3~Fz~S+g1w?itnaA5Fk$ati))Nq)LbwrX9Ub&R9rUq<|oA#WrZ@?N-oZbvtw?wzXgM7$)pi^`bmknrDXUy8Q#0NUjy-+PRs`hT@L(Eqd9d>Qrs81kw#2-a#tN}Hmv#?(4RDA*c!g|3!W zm55J6NqB2bQ_!=B5aHpKN^W|w?y`a+M1CzP{w5Pj+Nw{)n~okJ$C6CM7Cv2V8nP3z zZRvTDlEWEu{Le0@;on{BCi&79=N;oH{^xh{Ka=S+;(rWzWfkZBTSd4{gE9ymAmqqx z0sVc2&Z+{jTSI=HbJF&N=S*3O;ND2wJO~!?j7Y7n6@`W_x_*CsIT8QJ5uBX>I$r-@ z-i`la{eKjBN%+4OT*6=XA)k|oS7h8f0=Trb;M22}nhZSBJE;=ajoDU;Z2n#W{A^4{ ze^@R9RtN$+RQ~C$s1JTB@o=V_Mp@BwN=O%id;yb)kKVMTX!=`DMUMBgs}R-4)62K< zUv_Sd>-WVyv;cTU{5QRf@!u$N()CB4_RQE1a!mivCx`LhbUusz-zf4M=h@c=YS|mn z8r^u5_0p=#ptiRqbhE7Vf039k{(ELR&WyS62O_f3zNYNn0fUjh$y{tqn z?KK67l0H>6WJJ7`nMN%D-6Dp(#)`a+P8LWrHkl#M^R2SvMdEwuT+m%vi=oV3<# z(Z%`RG^!T`8a+d~&vpFtYr1DrdT1#BZA*hjPE-Uwl~x68qwr5Btk#X%^^y%>Cui@E zj_$JL;;oVtbDFx94aN?1bA8hy_TIWgT&1Kjnx4J}N`;xu5UQ5wSj*9}MgdAD8e=2+ z&zIBiU(qEv7yYNif3pMpPgawN|54;4tT6M>VXx9nJWODv7Mbe;UJd}b+Dd0J$ z;9ww!l!vwhU*a7@YY&=+NFmVQrt|1P6mny9w#z$njw*N@8o8tAK68H$Il#Z`vP?B} z#ZtFmN;)9I{AuL?|1zUk(7A-}g~uBJ{{H;$a<+`~zoW=K*Z*cX^MP}K$Ls%x z`hT^G^S|TBJAJLl5rdjfDdIy!XL&+*O>W%>R@q$zAC4lnl*{|pm>LX{P5s7ym@L2mfp3w@Nh>3O^f^_0|J6Sz z7(Ao@=W;cR@!uG7>h&*30klb>^D~ZdT>szc|6a}_|H~-ytFDIpKWcBgTHU_GhBI`? zyeHz9S{BIv4@rLR5#X#H7+%Qy<`|4mNl%2jzQ+bFv$9`TD=0#yJ^+6A$nCN$%&1I@YQAxTbFu(tWZj)U^7hc0Gp*vovwm!5KdII{#;W#CEeJKTk?C3G?6iNv$qWlTvRQ#hDlzgq+ zIn4ma`~Uur{g3hADDv6ZY@p&{&L5o{baXac5Re8*s<=Oowi!mYws53NHtByYJj zHPd>y(sWgpP>9!oMlvuAUu&P13ba2~o6ZT4N+jZ^JclZ9vKMK^IVUKOs#0Jdx#=Jr z)W_QN=nH0YN<)z;ItkV>+58Z>FaL}43(l|SnE#l^`rjyW()EYe{s1Q6u&D zOcAwKtp3T-R9{=Tl0#e<+CaI_n;W}b$t-lbpV8u=ebkyrb83y|J5VN_xmL>C@IqKK2B}^1>_$5 zbDg!&`7`i;U#^zZ82^nSr(A#6S(TK&uFgEh(fIEU|1bG}G5TZsrJD(7rhnQiLYG-q#VF!l7FB2tSvYA1;k% z!tr2N!XBDLyw_F5Q6+t!YdC;SnE&@>O!YmsoP>Xsb{rL)ITZB$_>cTgW69O7v5Nw; zevdfjf7^o5e_csJaD(}8@j<7aOaYb!q!ns_)_^k{)k9D`B^MM-dfjwNT-Az#2ic(h zH?)p%__#H;%69d{J&nM?2#EoqerO^c;>=-`*s2YRh?(t*i2uixlki{4Yjy7Oj~A!R z|KG9y%SGh>8AaZ6;-%UTc969tV3%O#7;>~bN)7;4wc=Vc=?3P0hY`KoiC|3T4m;pJ zamv?%Fn}3?LVqaJeZb{FJ_r5fe{}Fba$`AtG~|swypK2^M-K5{1?8WZ?7_5ed-?wO zZ@EDIKgNGgFZW&lVTLzZ*X@IjalHO_xc_;zjQgL*kzbmO&H!4nhQ18;=>q3dB}(I% zzAd?U$MK=@T-ieh=bDK_hxEPEUwKhPB0iSz-napT%Y6v1r;1O#(v|FxYtTaz}{%XmTI^ zGZ1woIeB~!JH}D_zqq^q^D^Rp6uAoY`mHfmY`7aI?fj0-W!UV59UyQ5Pz6M%C(r!e z5b_V@{DA`)#=qx&fH3u+Lbud(ms#eLDXgqRc`?_vMu(8?LRUehusC>_^w=k!pZom$ z-Q_;~w{$^x=Jy%)|3?0oapbh?&$XGex!{lbf0uXnzeoK)j=WX{a{d_Y9#%BnO7R!c zk*=GAKUfH{i)JN;jz0OJ1dQRL^2Tfaj7CL^z|ZYAJ9$l%R}B-tHQd;(Y$;m`5q}Pw9Rm5= zt|f?WD~-G|wm^V+VEAD6EQaLsTFK7qysHF7r7{89aX$p1W>tyZ!BYZQ6IWDWbPyQ{U4{|Ax>l=?jg+mZQ~UJWl% z?G2OPFdG)OeNlXtefF!MCJsiZ-p(T&wT=aa5=!gCui$i zn(uGQQTsnVjQt?@L4QEJ-iiY7WxF8i1TRg5QG)By9BN z!LeF59kAS@OL&(<3gv!Y7fz}H+hAltu|+T15)r^!Qn>Kk=OIdzC{dzBi4rABlqgZ6 VM2Qk5N|e8&{0~Oy30VNh008@i&V~R0 literal 0 HcmV?d00001 diff --git a/tests/test_alipcs.py b/tests/test_alipcs.py new file mode 100644 index 0000000..b52947d --- /dev/null +++ b/tests/test_alipcs.py @@ -0,0 +1,226 @@ +import os +from pathlib import Path +import time +import random + +import pytest +import qrcode + +from alipcs_py.alipcs import AliPCS, AliOpenPCS, AliOpenAuth + +from tests.datas import REFRESH_TOKEN, Datas + + +class TestAliPCS: + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_get_token(self, alipcs: AliPCS): + info = alipcs.get_token() + assert info["access_token"] != "" + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_refresh(self, alipcs: AliPCS): + info = alipcs.refresh() + assert info["access_token"] != "" + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_get_drive_info(self, alipcs: AliPCS): + assert alipcs.device_id != "" + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_create_session(self, alipcs: AliPCS): + info = alipcs.create_session() + assert info["result"] and info["success"] + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_path_traceback(self, alipcs: AliPCS, datas: Datas): + local_dir = datas.local_dir + local_paths = datas.local_paths + remote_dir = datas.remote_dir + + local_path = random.choice(local_paths) + remote_path = Path(remote_dir) / local_path[len(local_dir) + 1 :] + remote_path_posix = remote_path.as_posix() + file_id = alipcs.meta_by_path(remote_path_posix)["file_id"] + + info = alipcs.path_traceback(file_id) + wanted_path = Path("/", *[p["name"] for p in info["items"][::-1]]) + assert wanted_path == remote_path + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_meta_by_path(self, alipcs: AliPCS, datas: Datas): + remote_dir = datas.remote_dir + info = alipcs.meta_by_path(remote_dir) + assert info["file_id"] != "" + assert info["name"] == os.path.basename(remote_dir) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_meta(self, alipcs: AliPCS, datas: Datas): + pcs_file = datas.remote_dir_pcs_file + info = alipcs.meta(pcs_file.file_id) + assert info["file_id"] == pcs_file.file_id + assert info["name"] == pcs_file.name + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_exists(self, alipcs: AliPCS, datas: Datas): + pcs_file = datas.remote_dir_pcs_file + assert alipcs.exists(pcs_file.file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_is_dir(self, alipcs: AliPCS, datas: Datas): + pcs_file = datas.remote_dir_pcs_file + assert alipcs.is_dir(pcs_file.file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list(self, alipcs: AliPCS, datas: Datas): + local_dir = datas.local_dir + pcs_file = datas.remote_dir_pcs_file + filenames = set(os.listdir(local_dir)) + info = alipcs.list(pcs_file.file_id) + for v in info["items"]: + assert v["name"] in filenames + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_walk(self, alipcs: AliPCS, datas: Datas): + pcs_file = datas.remote_dir_pcs_file + alipcs.walk(pcs_file.file_id) + + # More tests in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_create_file(self, alipcs: AliPCS): + pass + # Tested in conftest.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_rapid_upload_file(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_search(self, alipcs: AliPCS, datas: Datas): + time.sleep(10) # Wait for the file to be indexed + + local_paths = datas.local_paths + local_path = random.choice(local_paths) + name = os.path.basename(local_path) + info = alipcs.search(name) + assert any(v["name"] == name for v in info["items"]) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_makedir(self, alipcs: AliPCS): + pass + # Tested in conftest.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_move(self, alipcs: AliPCS, datas: Datas): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_rename(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_copy(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_remove(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_share(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list_shared(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_cancel_shared(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_get_share_token(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_shared_info(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list_shared_files(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_transfer_shared_files(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_shared_file_download_url(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_user(self, alipcs: AliPCS): + info = alipcs.user_info() + assert info["user_id"] != "" + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_download_link(self, alipcs: AliPCS, datas: Datas): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_file_stream(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_shared_file_stream(self, alipcs: AliPCS): + pass + # Tested in test_alipcsapi.py + + # def test_qrcode_link(self, alipcs: AliPCS): + # ali_auth = AliOpenAuth(client_server=ALIYUNDRIVE_OPENAPI_AUTH_DOMAIN) + # info = ali_auth.get_qrcode_info() + # print(info) + # + # def test_qrcode_auth(self, alipcs: AliPCS): + # ali_auth = AliOpenAuth(client_server=ALIYUNDRIVE_OPENAPI_AUTH_DOMAIN) + # + # # Get qrcode info + # info = ali_auth.get_qrcode_info() + # print(info) + # sid = info["sid"] + # + # qrcode_url = f"https://www.aliyundrive.com/o/oauth/authorize?sid={sid}" + # + # qr = qrcode.QRCode() + # qr.add_data(qrcode_url) + # f = io.StringIO() + # qr.print_ascii(out=f, tty=False, invert=True) + # f.seek(0) + # print(f.read()) + # + # while True: + # info = ali_auth.scan_status(sid) + # print(info) + # if info["status"] == "LoginSuccess": + # auth_code = info["authCode"] + # break + # time.sleep(2) + # + # info = ali_auth.get_refresh_token(auth_code) + # + # print(info) diff --git a/tests/test_alipcsapi.py b/tests/test_alipcsapi.py new file mode 100644 index 0000000..de6cc43 --- /dev/null +++ b/tests/test_alipcsapi.py @@ -0,0 +1,345 @@ +from pathlib import Path, PosixPath +import os +import random +import time + +from alipcs_py import AliPCSApi + +import pytest +from rich import print + +from tests.datas import REFRESH_TOKEN, Datas + + +class TestAliPCSApi: + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_refresh_token(self, alipcsapi: AliPCSApi, datas: Datas): + assert alipcsapi.refresh_token != "" + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_access_token(self, alipcsapi: AliPCSApi, datas: Datas): + assert alipcsapi.access_token != "" + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_expire_time(self, alipcsapi: AliPCSApi, datas: Datas): + assert alipcsapi.expire_time > 0 + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_user_id(self, alipcsapi: AliPCSApi, datas: Datas): + assert alipcsapi.user_id != "" + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_device_id(self, alipcsapi: AliPCSApi, datas: Datas): + assert alipcsapi.device_id != "" + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_default_drive_id(self, alipcsapi: AliPCSApi, datas: Datas): + assert alipcsapi.default_drive_id != "" + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_path_traceback(self, alipcsapi: AliPCSApi, datas: Datas): + remote_path = random.choice(datas.remote_paths) + pcs_file = alipcsapi.meta_by_path(remote_path) + assert pcs_file is not None + files = alipcsapi.path_traceback(pcs_file.file_id) + assert remote_path == files[0].path + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_meta_by_path(self, alipcsapi: AliPCSApi, datas: Datas): + remote_path = random.choice(datas.remote_paths) + pcs_file = alipcsapi.meta_by_path(remote_path) + assert pcs_file is not None + assert pcs_file.path == remote_path + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_meta(self, alipcsapi: AliPCSApi, datas: Datas): + pcs_file = datas.remote_dir_pcs_file + pf = alipcsapi.meta(pcs_file.file_id) + assert pf is not None + assert pf.name == pf.path + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_get_file(self, alipcsapi: AliPCSApi, datas: Datas): + remote_path = random.choice(datas.remote_paths) + pcs_file = alipcsapi.get_file(remotepath=remote_path) + assert pcs_file is not None + assert pcs_file.path == remote_path + + pcs_file = datas.remote_dir_pcs_file + pf = alipcsapi.get_file(file_id=pcs_file.file_id) + assert pf is not None + assert pf.name == pf.path + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_exists(self, alipcsapi: AliPCSApi, datas: Datas): + pcs_file = datas.remote_dir_pcs_file + assert alipcsapi.exists(pcs_file.file_id) + assert not alipcsapi.exists(pcs_file.file_id[::-1]) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_is_file(self, alipcsapi: AliPCSApi, datas: Datas): + pcs_file = datas.remote_dir_pcs_file + assert not alipcsapi.is_file(pcs_file.file_id) + + remote_path = random.choice(datas.remote_paths) + pcs_file = alipcsapi.meta_by_path(remote_path) + assert pcs_file is not None + assert alipcsapi.is_file(pcs_file.file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_is_dir(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # Same as test_is_file + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list(self, alipcsapi: AliPCSApi, datas: Datas): + pcs_file = datas.remote_dir_pcs_file + sub_pcs_files, _ = alipcsapi.list(pcs_file.file_id) + local_dir = datas.local_dir + for sub_pcs_file in sub_pcs_files: + assert sub_pcs_file.path == sub_pcs_file.name + assert Path(local_dir, sub_pcs_file.path).exists() + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list_iter(self, alipcsapi: AliPCSApi, datas: Datas): + pcs_file = datas.remote_dir_pcs_file + sub_pcs_files = list(alipcsapi.list_iter(pcs_file.file_id, recursive=True, include_dir=True)) + local_dir = datas.local_dir + for sub_pcs_file in sub_pcs_files: + assert not sub_pcs_file.path.startswith(pcs_file.name) + assert Path(local_dir, PosixPath(sub_pcs_file.path)).exists() + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_path(self, alipcsapi: AliPCSApi, datas: Datas): + remote_path = sorted(datas.remote_paths, key=lambda x: len(x))[-1] + pcs_file = alipcsapi.path(remote_path) + assert pcs_file is not None + assert remote_path == pcs_file.path + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_paths(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # Tested in test_path + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list_path_iter(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # Deprecated + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list_path(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # Deprecated + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_walk(self, alipcsapi: AliPCSApi, datas: Datas): + remote_dir_pcs_file = datas.remote_dir_pcs_file + remote_dir = datas.remote_dir + local_dir = datas.local_dir + remote_paths = set(datas.remote_paths) + wanted_paths = set() + for pcs_file in alipcsapi.walk(remote_dir_pcs_file.file_id): + assert Path(local_dir, pcs_file.path).exists() + if pcs_file.is_file: + wanted_paths.add(remote_dir + "/" + pcs_file.path) + assert wanted_paths == remote_paths + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_create_file(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # Tested in test_commands.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_prepare_file(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # Tested in test_commands.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_get_upload_url(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # Tested in test_commands.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_rapid_upload_file(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # Tested in test_commands.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_upload_slice(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # Tested in test_commands.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_upload_complete(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # Tested in test_commands.py + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_search(self, alipcsapi: AliPCSApi, datas: Datas): + remote_path = random.choice(datas.remote_paths) + name = os.path.basename(remote_path) + time.sleep(10) # Wait for the file to be indexed + assert any(pcs_file.name == name for pcs_file in alipcsapi.search(name)[0]) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_search_all(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # Tested in test_search + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_makedir(self, alipcsapi: AliPCSApi, datas: Datas): + name = "test_makedir1" + pcs_file = alipcsapi.makedir("root", name) + assert pcs_file is not None + assert pcs_file.name == name + alipcsapi.remove(pcs_file.file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_makedir_path(self, alipcsapi: AliPCSApi, datas: Datas): + path = "/test_makedir_path2/test_makedir_path3/test_makedir_path4" + pcs_files = alipcsapi.makedir_path(path) + try: + parts = path.split("/") + for i in range(1, len(parts)): + assert pcs_files[i - 1].path == "/".join(parts[: len(parts) - i + 1]) + finally: + alipcsapi.remove(pcs_files[-1].file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_move(self, alipcsapi: AliPCSApi, datas: Datas): + path = "/test_move/test_move1/test_move2" + pcs_files = alipcsapi.makedir_path(path) + try: + result = alipcsapi.move(pcs_files[0].file_id, pcs_files[-1].file_id) + assert all(result) + + assert alipcsapi.get_file(remotepath="/test_move/test_move2") is not None + assert alipcsapi.get_file(remotepath="/test_move/test_move1/test_move2") is None + finally: + alipcsapi.remove(pcs_files[-1].file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_rename(self, alipcsapi: AliPCSApi, datas: Datas): + path = "/test_rename/test_rename1/test_rename2" + pcs_files = alipcsapi.makedir_path(path) + try: + pf = alipcsapi.rename(pcs_files[0].file_id, "test_rename3") + assert pf is not None + assert pf.name == "test_rename3" + assert alipcsapi.get_file(remotepath=path) is None + assert alipcsapi.get_file(remotepath=path.replace("2", "3")) is not None + finally: + pf = alipcsapi.get_file(remotepath="/test_rename") + if pf is not None: + alipcsapi.remove(pf.file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_copy(self, alipcsapi: AliPCSApi, datas: Datas): + path = "/test_copy/test_copy1/test_copy2" + pcs_files = alipcsapi.makedir_path(path) + try: + new_files = alipcsapi.copy(pcs_files[0].file_id, pcs_files[-1].file_id) + assert len(new_files) == 1 + + assert alipcsapi.get_file(remotepath="/test_copy/test_copy2") is not None + finally: + alipcsapi.remove(pcs_files[-1].file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_remove(self, alipcsapi: AliPCSApi, datas: Datas): + path = "/test_remove/test_remove1/test_remove2" + pcs_files = alipcsapi.makedir_path(path) + try: + assert alipcsapi.remove(pcs_files[0].file_id) + assert alipcsapi.get_file(remotepath=path) is None + finally: + alipcsapi.remove(pcs_files[-1].file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_share(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_is_shared_valid(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list_shared(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list_shared_all(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_cancel_shared(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_get_share_token(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_shared_info(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_transfer_shared_files(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_shared_file_download_url(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_user_info(self, alipcsapi: AliPCSApi, datas: Datas): + info = alipcsapi.user_info() + assert info.user_id != "" + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_download_link(self, alipcsapi: AliPCSApi, datas: Datas): + remote_path = random.choice(datas.remote_paths) + pcs_file = alipcsapi.meta_by_path(remote_path) + assert pcs_file is not None + link = alipcsapi.download_link(pcs_file.file_id) + assert link is not None + assert link.download_url or link.url + assert not link.expires() + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_update_download_url(self, alipcsapi: AliPCSApi, datas: Datas): + remote_path = random.choice(datas.remote_paths) + pcs_file = alipcsapi.meta_by_path(remote_path) + assert pcs_file is not None + pcs_file = alipcsapi.update_download_url(pcs_file) + assert not pcs_file.download_url_expires() + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_file_stream(self, alipcsapi: AliPCSApi, datas: Datas): + remote_path = random.choice(datas.remote_paths) + remote_dir = datas.remote_dir + local_path = Path(datas.local_dir, PosixPath(remote_path[len(remote_dir) + 1 :])) + pcs_file = alipcsapi.meta_by_path(remote_path) + assert pcs_file is not None + stream = alipcsapi.file_stream(pcs_file.file_id) + assert stream is not None + assert stream.readable() + assert stream.seekable() + content = stream.read() + assert content is not None + assert len(content) == pcs_file.size + assert content == local_path.read_bytes() + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_shared_file_stream(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # share api changed, need to update diff --git a/tests/test_commands.py b/tests/test_commands.py new file mode 100644 index 0000000..3655f09 --- /dev/null +++ b/tests/test_commands.py @@ -0,0 +1,332 @@ +import os +import sys +import random +import time +import io +from pathlib import Path, PosixPath + +from alipcs_py.alipcs import AliPCSApi +from alipcs_py.commands.list_files import list_files +from alipcs_py.commands.search import search +from alipcs_py.commands.file_operators import makedir, move, rename, copy, remove +from alipcs_py.commands.upload import upload, from_tos, _rapid_upload +from alipcs_py.commands.share import ( + list_shared, + share_files, + cancel_shared, + save_shared, + list_shared_files, +) +from alipcs_py.commands.user import show_user_info +from alipcs_py.commands.download import download, Downloader +from alipcs_py.commands.server import start_server +from alipcs_py.commands.crypto import decrypt_file +from alipcs_py.common.crypto import calc_proof_code, calc_sha1 + +import pytest +from faker import Faker + +from alipcs_py.common.io import EncryptType, reset_encrypt_io + +from tests.datas import REFRESH_TOKEN, Datas + + +fake = Faker() + + +class CaptureStdout: + def __init__(self): + self.sys_stdout = sys.stdout + self.io = io.StringIO() + sys.stdout = self.io + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + sys.stdout = self.sys_stdout + + def get_output(self): + return self.io.getvalue() + + +class TestCommands: + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list_file(self, alipcsapi: AliPCSApi, datas: Datas): + remote_dir = datas.remote_dir + + with CaptureStdout() as cs: + list_files( + alipcsapi, + remote_dir, + show_size=True, + recursive=False, + sifters=[], + highlight=True, + show_file_id=True, + show_date=True, + ) + + output = cs.get_output() + part1, part2 = remote_dir.rsplit("/", 1) + assert part1 in output and part2 in output + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_search(self, alipcsapi: AliPCSApi, datas: Datas): + remote_path = random.choice(datas.remote_paths) + name = os.path.basename(remote_path) + time.sleep(10) # Wait for the file to be indexed + + with CaptureStdout() as cs: + search(alipcsapi, name) + + output = cs.get_output() + assert name in output + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_makedir(self, alipcsapi: AliPCSApi): + path = "/test_makedir_cmd/test_makedir_cmd1/test_makedir_cmd2" + with CaptureStdout() as cs: + makedir(alipcsapi, path, show=True) + + output = cs.get_output() + try: + assert alipcsapi.get_file(remotepath=path) is not None + assert path in output + finally: + remove(alipcsapi, "/".join(path.split("/")[:2])) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_move(self, alipcsapi: AliPCSApi): + from_path = "/test_move_cmd/test_move_cmd1/test_move_cmd2" + to_path = "/test_move_cmd/tmp/test_move_cmd3" + from_paths = alipcsapi.makedir_path(from_path) + + with CaptureStdout() as cs: + move(alipcsapi, from_path, to_path, show=True) + + output = cs.get_output() + try: + assert alipcsapi.get_file(remotepath=to_path) is not None + assert alipcsapi.get_file(remotepath=from_path) is None + assert to_path in output + finally: + alipcsapi.remove(from_paths[-1].file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_rename(self, alipcsapi: AliPCSApi): + path = "/test_rename_cmd/test_rename_cmd1" + new_name = "test_rename_cmd2" + from_paths = alipcsapi.makedir_path(path) + + with CaptureStdout() as cs: + rename(alipcsapi, path, new_name, show=True) + + output = cs.get_output() + try: + assert alipcsapi.get_file(remotepath="/".join(path.split("/")[:-1] + [new_name])) is not None + assert new_name in output + finally: + alipcsapi.remove(from_paths[-1].file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_copy(self, alipcsapi: AliPCSApi): + from_path = "/test_copy_cmd/test_copy_cmd1/test_copy_cmd2" + to_path = "/test_copy_cmd/tmp" + from_paths = alipcsapi.makedir_path(from_path) + + with CaptureStdout() as cs: + copy(alipcsapi, from_path, to_path, show=True) + + output = cs.get_output() + try: + pcs_file = alipcsapi.get_file(remotepath=to_path + "/test_copy_cmd2") + assert pcs_file is not None + assert pcs_file.file_id in output + finally: + alipcsapi.remove(from_paths[-1].file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_remove(self, alipcsapi: AliPCSApi): + path = "/test_remove_cmd" + paths = alipcsapi.makedir_path(path) + remove(alipcsapi, path) + assert not alipcsapi.exists(paths[0].file_id) + assert alipcsapi.exists_in_trash(paths[0].file_id) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_upload(self, alipcsapi: AliPCSApi, tmp_path: str): + remotedir = "/test_upload_cmd" + file_size = 1024 * 1024 * 10 # 10MB + content = os.urandom(file_size) + sha1 = calc_sha1(content) + name = "test_upload_cmd" + local_path = Path(tmp_path) / name + with open(local_path, "wb") as f: + f.write(content) + + # Upload file + upload(alipcsapi, from_to_list=from_tos([local_path], remotedir), show_progress=False) + try: + pcs_file = alipcsapi.get_file(remotepath=remotedir + "/" + name) + assert pcs_file is not None + assert pcs_file.size == file_size + assert pcs_file.rapid_upload_info is not None + assert pcs_file.rapid_upload_info.content_hash.lower() == sha1.lower() + finally: + remove(alipcsapi, remotedir) + + # Rapid Upload + file_io = open(local_path, "rb") + slice1k_bytes = file_io.read(1024) + reset_encrypt_io(file_io) + slice1k_hash = calc_sha1(slice1k_bytes) + + remote_pcs_file = alipcsapi.makedir_path(remotedir + "/tmp")[0] + + pcs_prepared_file = alipcsapi.prepare_file( + name, + remote_pcs_file.file_id, + file_size, + slice1k_hash, + part_number=1, + check_name_mode="overwrite", + ) + content_hash = calc_sha1(file_io) + proof_code = calc_proof_code(file_io, file_size, alipcsapi.access_token) + + try: + assert pcs_prepared_file.can_rapid_upload() + assert _rapid_upload( + alipcsapi, + local_path.as_posix(), + name, + remote_pcs_file.file_id, + content_hash, + proof_code, + file_size, + check_name_mode="overwrite", + task_id=None, + ) + assert alipcsapi.get_file(remotepath=remotedir + "/tmp/" + name) is not None + finally: + remove(alipcsapi, remotedir) + + # Encrypt Upload + password = b"1234" + for enc_type in EncryptType: + upload( + alipcsapi, + from_to_list=from_tos([local_path], remotedir), + encrypt_password=password, + encrypt_type=enc_type, + show_progress=False, + ) + try: + pcs_file = alipcsapi.get_file(remotepath=remotedir + "/" + name) + assert pcs_file is not None + download( + alipcsapi, [pcs_file.path], localdir=Path(tmp_path, enc_type.value), encrypt_password=password + ) + target_path = Path(tmp_path, enc_type.value, pcs_file.name) + assert target_path.exists() + target_sha1 = calc_sha1(target_path.read_bytes()) + assert target_sha1 == sha1 + finally: + remove(alipcsapi, remotedir) + + # Upload directory + upload(alipcsapi, from_to_list=from_tos([tmp_path], remotedir), show_progress=False) + try: + pcs_file = alipcsapi.get_file(remotepath=remotedir + "/" + os.path.basename(tmp_path) + "/" + name) + assert pcs_file is not None + assert pcs_file.size == file_size + assert pcs_file.rapid_upload_info is not None + assert pcs_file.rapid_upload_info.content_hash.lower() == sha1.lower() + finally: + remove(alipcsapi, remotedir) + os.remove(local_path) + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list_shared(self, alipcsapi: AliPCSApi): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_share(self, alipcsapi: AliPCSApi): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_cancel_shared(self, alipcsapi: AliPCSApi): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_save_shared(self, alipcsapi: AliPCSApi): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_list_shared_files(self, alipcsapi: AliPCSApi): + pass + # share api changed, need to update + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_show_user_info(self, alipcsapi: AliPCSApi): + with CaptureStdout() as cs: + show_user_info(alipcsapi) + + output = cs.get_output() + assert alipcsapi.refresh_token in output + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_download(self, alipcsapi: AliPCSApi, datas: Datas, tmp_path): + # Download file + remote_path = random.choice(datas.remote_paths) + download(alipcsapi, [remote_path], localdir=tmp_path, downloader=Downloader.me, show_progress=False) + pcs_file = alipcsapi.get_file(remotepath=remote_path) + assert pcs_file is not None + assert pcs_file.rapid_upload_info is not None + local_path = Path(tmp_path) / os.path.basename(remote_path) + assert os.path.exists(local_path) + sha1 = calc_sha1(local_path.open("rb")) + assert sha1.lower() == pcs_file.rapid_upload_info.content_hash.lower() + + # Download directory + remote_dir = datas.remote_dir + download( + alipcsapi, + [remote_dir], + localdir=tmp_path, + downloader=Downloader.me, + recursive=True, + show_progress=False, + ) + + remote_dir_name = os.path.basename(remote_dir) + remote_pcs_file = datas.remote_dir_pcs_file + pcs_files = alipcsapi.walk(remote_pcs_file.file_id) + for pcs_file in pcs_files: + if pcs_file.is_dir: + continue + local_path = Path(tmp_path) / remote_dir_name / PosixPath(pcs_file.path) + assert local_path.exists() + sha1 = calc_sha1(local_path.open("rb")) + assert pcs_file.rapid_upload_info is not None + assert sha1.lower() == pcs_file.rapid_upload_info.content_hash.lower() + + @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + def test_play(self, alipcsapi: AliPCSApi, datas: Datas): + pass + # No support at IC + + # + # @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + # def test_http_server(self, alipcsapi: AliPCSApi, datas: Datas): + # print() + # start_server(alipcsapi, "/") + # + # @pytest.mark.skipif(not REFRESH_TOKEN, reason="No REFRESH_TOKEN") + # def test_decrypt_file(self, alipcsapi: AliPCSApi, datas: Datas): + # decrypt_file("f60m", "f60m_dec", "CK-QEpQ)T@@P{kXV/GGw") diff --git a/tests/test_common.py b/tests/test_common.py index f5c1b22..c5e0250 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,12 +1,10 @@ import time import os import io -import sys import subprocess import requests -from alipcs_py.common.concurrent import Executor from alipcs_py.common import constant from alipcs_py.common.number import u64_to_u8x8, u8x8_to_u64 from alipcs_py.common.path import join_path @@ -22,7 +20,6 @@ AES256CBCEncryptIO, to_decryptio, rapid_upload_params, - EncryptType, ) from alipcs_py.common.crypto import ( generate_key_iv, @@ -31,7 +28,6 @@ random_bytes, _md5_cmd, calc_file_md5, - calc_md5, SimpleCryptography, ChaCha20Cryptography, AES256CBCCryptography, @@ -431,21 +427,3 @@ def test_human_size(): s_int = human_size_to_int(s_str) assert s == s_int - - -def test_executor(): - def f(n): - return n - - with Executor(max_workers=1) as executor: - r = executor.submit(f, 1) - assert r == 1 - - futs = [] - with Executor(max_workers=2) as executor: - fut = executor.submit(f, 1) - futs.append(fut) - - for fut in futs: - r = fut.result() - assert r == 1 From b37e857ae9c71ec71438680af4525e88eeed7d8a Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 22:56:16 +0800 Subject: [PATCH 26/32] Add MANIFEST.in --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..bc8488f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +global-exclude **/__pycache__/** +global-exclude *.pyc From 593fcb70bacc633d76b8650aed21336e07fd6f68 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 23:02:45 +0800 Subject: [PATCH 27/32] Fix type --- tests/datas.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/datas.py b/tests/datas.py index fe1f241..5c3a83f 100644 --- a/tests/datas.py +++ b/tests/datas.py @@ -1,5 +1,6 @@ import os from dataclasses import dataclass +from typing import List from alipcs_py.alipcs.inner import PcsFile @@ -9,7 +10,7 @@ @dataclass class Datas: local_dir: str - local_paths: list[str] + local_paths: List[str] remote_dir: str remote_dir_pcs_file: PcsFile - remote_paths: list[str] + remote_paths: List[str] From a1bb0108819ad2671c0811624e0a9cb4ce0137e3 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Mon, 8 Apr 2024 23:16:47 +0800 Subject: [PATCH 28/32] Compat the `Annotated` for Python<3.9 --- alipcs_py/commands/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alipcs_py/commands/server.py b/alipcs_py/commands/server.py index 543fd3a..7d5ed28 100644 --- a/alipcs_py/commands/server.py +++ b/alipcs_py/commands/server.py @@ -1,9 +1,9 @@ import sys -if sys.version_info > (3, 8): - from typing import Annotated -else: +if sys.version_info < (3, 9): from typing_extensions import Annotated +else: + from typing import Annotated from typing import Optional, Dict, Any From 79a75bf10e24598748bed15c763c5daffb5bb027 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Tue, 9 Apr 2024 22:01:11 +0800 Subject: [PATCH 29/32] Change `RangeRequestIO` callback function signature --- alipcs_py/common/io.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/alipcs_py/common/io.py b/alipcs_py/common/io.py index e651544..b49d008 100644 --- a/alipcs_py/common/io.py +++ b/alipcs_py/common/io.py @@ -1011,13 +1011,26 @@ def _split_chunk(self, start: int, end: int) -> List[Tuple[int, int]]: class RangeRequestIO(IO): + """Request IO with range header + + Args: + method (str): HTTP method + url (str): URL + headers (Optional[Dict[str, str]]): HTTP headers + max_chunk_size (int): The max chunk size + callback (Optional[Callable[[int], None]]): Callback function for progress monitor, + the argument is the current offset. + encrypt_password (bytes): Encrypt password + **kwargs: Other kwargs for request + """ + def __init__( self, method: str, url: str, headers: Optional[Dict[str, str]] = None, max_chunk_size: int = DEFAULT_MAX_CHUNK_SIZE, - callback: Optional[Callable[..., None]] = None, + callback: Optional[Callable[[int], Any]] = None, encrypt_password: bytes = b"", **kwargs, ): @@ -1056,14 +1069,14 @@ def read(self, size: int = -1) -> Optional[bytes]: start, end = self._offset, self._offset + size - buf = b"" - for b in self._auto_decrypt_request.read((start, end)): - buf += b - self._offset += len(b) + buffer = b"" + for buf in self._auto_decrypt_request.read((start, end)): + buffer += buf + self._offset += len(buf) # Call callback if self._callback: self._callback(self._offset) - return buf + return buffer def read_iter(self, size: int = -1) -> Iterable[bytes]: if size == 0: From f5d9369e2e871b58c4cdb64d52ba5465d3a3f81e Mon Sep 17 00:00:00 2001 From: PeterDing Date: Tue, 9 Apr 2024 22:03:48 +0800 Subject: [PATCH 30/32] Change `upload_slice` argument name and signature `callback: Optional[Callable[[MultipartEncoderMonitor], None]]` to `callback_for_monitor: Optional[Callable[[int], Any]]` --- alipcs_py/alipcs/api.py | 10 ++++------ alipcs_py/alipcs/pcs.py | 11 +++++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/alipcs_py/alipcs/api.py b/alipcs_py/alipcs/api.py index 14de103..9b8c791 100644 --- a/alipcs_py/alipcs/api.py +++ b/alipcs_py/alipcs/api.py @@ -1,4 +1,4 @@ -from typing import Optional, List, Tuple, Dict, Union, DefaultDict, Iterable, Callable, IO +from typing import Any, Optional, List, Tuple, Dict, Union, DefaultDict, Iterable, Callable, IO from threading import Lock from collections import defaultdict from copy import deepcopy @@ -548,15 +548,13 @@ def rapid_upload_file( filename, dir_id, size, content_hash=content_hash, proof_code=proof_code, check_name_mode=check_name_mode ) - def upload_slice( - self, io: IO, url: str, callback: Optional[Callable[[MultipartEncoderMonitor], None]] = None - ) -> None: + def upload_slice(self, io: IO, url: str, callback_for_monitor: Optional[Callable[[int], Any]] = None) -> None: """Upload an io as a slice - callable: the callback for monitoring uploading progress + callable: the callback for monitor """ - self._alipcs.upload_slice(io, url, callback=callback) + self._alipcs.upload_slice(io, url, callback_for_monitor=callback_for_monitor) def upload_complete(self, file_id: str, upload_id: str) -> PcsFile: """Tell server that all slices have been uploaded diff --git a/alipcs_py/alipcs/pcs.py b/alipcs_py/alipcs/pcs.py index d9c0b82..cccc630 100644 --- a/alipcs_py/alipcs/pcs.py +++ b/alipcs_py/alipcs/pcs.py @@ -701,20 +701,19 @@ def rapid_upload_file(self, filename: str, dir_id: str, size: int, content_hash: return self.create_file(filename, dir_id, size, content_hash=content_hash, proof_code=proof_code) - def upload_slice( - self, io: IO, url: str, callback: Optional[Callable[[MultipartEncoderMonitor], None]] = None - ) -> None: + def upload_slice(self, io: IO, url: str, callback_for_monitor: Optional[Callable[[int], Any]] = None) -> None: """Upload the content of io to remote url""" - cio = ChunkIO(io, total_len(io)) - monitor = MultipartEncoderMonitor(cio, callback=callback) + data = ChunkIO(io, total_len(io)) + if callback_for_monitor is not None: + data = MultipartEncoderMonitor(data, callback=lambda monitor: callback_for_monitor(monitor.bytes_read)) session = requests.Session() session.request( "PUT", url, headers=dict(PCS_HEADERS), - data=monitor, + data=data, timeout=(3, 9), # (connect timeout, read timeout) ) From c96336db3e8f653c9168ef0fb29fb78f6dbc1112 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Tue, 9 Apr 2024 22:07:41 +0800 Subject: [PATCH 31/32] Name the callback function for progress monitor as `callback_for_monitor` --- alipcs_py/commands/download.py | 15 +++++-- alipcs_py/commands/upload.py | 78 ++++++++++++++++++++++------------ 2 files changed, 64 insertions(+), 29 deletions(-) diff --git a/alipcs_py/commands/download.py b/alipcs_py/commands/download.py index 2741c30..9b480f2 100644 --- a/alipcs_py/commands/download.py +++ b/alipcs_py/commands/download.py @@ -1,4 +1,4 @@ -from typing import Iterable, Optional, List, Sequence, Tuple, Union +from typing import Any, Callable, Iterable, Optional, List, Sequence, Tuple, Union from concurrent.futures import ThreadPoolExecutor, as_completed from enum import Enum from pathlib import Path @@ -65,6 +65,7 @@ def download( max_retries: int = 2, out_cmd: bool = False, encrypt_password: bytes = b"", + callback_for_monitor: Optional[Callable[[int], Any]] = None, ): global DEFAULT_DOWNLOADER if not self.which(): @@ -80,6 +81,7 @@ def download( show_progress=show_progress, max_retries=max_retries, encrypt_password=encrypt_password, + callback_for_monitor=callback_for_monitor, ) shutil.move(localpath_tmp, localpath) return @@ -149,6 +151,7 @@ def _me_download( show_progress: bool = False, max_retries: int = 2, encrypt_password: bytes = b"", + callback_for_monitor: Optional[Callable[[int], Any]] = None, ): headers = { "Referer": "https://www.aliyundrive.com/", @@ -166,19 +169,20 @@ def done_callback(): def monitor_callback(offset: int): if task_id is not None: - _progress.update(task_id, completed=offset + 1) + _progress.update(task_id, completed=offset) def except_callback(err): reset_progress_task(task_id) if isinstance(chunk_size, str): chunk_size = human_size_to_int(chunk_size) + io = RangeRequestIO( "GET", url, headers=headers, max_chunk_size=chunk_size, - callback=monitor_callback, + callback=monitor_callback if callback_for_monitor is None else callback_for_monitor, encrypt_password=encrypt_password, ) @@ -307,6 +311,7 @@ def download_file( max_retries: int = 2, out_cmd: bool = False, encrypt_password: bytes = b"", + callback_for_monitor: Optional[Callable[[int], Any]] = None, ) -> None: """Download a `remote_file` to the `localdir` @@ -324,6 +329,9 @@ def download_file( max_retries (int, optional): The max retries of download. Defaults to 2. out_cmd (bool, optional): Whether print out the command. Defaults to False. encrypt_password (bytes, optional): The password to decrypt the file. Defaults to b"". + callback_for_monitor (Callable[[int], Any], optional): The callback function for monitor. Defaults to None. + The callback function should accept one argument which is the count of bytes downloaded. + The callback function is only passed to the `MeDownloader` downloader. """ if isinstance(downloader, str): @@ -396,6 +404,7 @@ def download_file( max_retries=max_retries, out_cmd=out_cmd, encrypt_password=encrypt_password, + callback_for_monitor=callback_for_monitor, ) except Exception as origin_err: msg = f'Download "{remote_pcs_file.path}" (file_id = "{remote_pcs_file.file_id}") to "{localpath}" failed. error: {origin_err}' diff --git a/alipcs_py/commands/upload.py b/alipcs_py/commands/upload.py index 0858b3e..4543d26 100644 --- a/alipcs_py/commands/upload.py +++ b/alipcs_py/commands/upload.py @@ -1,4 +1,4 @@ -from typing import Callable, Optional, List, Sequence, Tuple, IO, Union +from typing import Any, Callable, Optional, List, Sequence, Tuple, IO, Union import os import time import math @@ -11,17 +11,21 @@ from alipcs_py.alipcs import AliPCSApi, FromTo from alipcs_py.alipcs.pcs import CheckNameMode from alipcs_py.common import constant -from alipcs_py.common.path import PathType, is_file, exists, posix_path_basename, posix_path_dirname +from alipcs_py.common.path import PathType, exists, posix_path_basename, posix_path_dirname from alipcs_py.common.event import KeyHandler, KeyboardMonitor from alipcs_py.common.constant import CPU_NUM from alipcs_py.common.concurrent import retry -from alipcs_py.common.progress_bar import _progress, progress_task_exists, remove_progress_task, reset_progress_task +from alipcs_py.common.progress_bar import ( + _progress, + init_progress_bar, + progress_task_exists, + remove_progress_task, + reset_progress_task, +) from alipcs_py.common.crypto import calc_sha1, calc_proof_code from alipcs_py.common.io import total_len, EncryptType, reset_encrypt_io from alipcs_py.commands.log import get_logger -from requests_toolbelt import MultipartEncoderMonitor - from rich.progress import TaskID from rich import print @@ -153,11 +157,7 @@ def upload( futures = [] with ThreadPoolExecutor(max_workers=max_workers) as executor: for idx, from_to in enumerate(from_to_list): - task_id = None - if show_progress: - task_id = _progress.add_task("upload", start=False, title=from_to[0]) - - logger.debug("`upload_many`: Upload: index: %s, task_id: %s", idx, task_id) + logger.debug("`upload_many`: Upload: index: %s", idx) retry_upload_file = retry( max_retries, @@ -178,7 +178,7 @@ def upload( encrypt_type=encrypt_type, slice_size=slice_size, only_use_rapid_upload=only_use_rapid_upload, - task_id=task_id, + show_progress=show_progress, ) futures.append(fut) @@ -271,8 +271,8 @@ def upload_file( encrypt_type: EncryptType = EncryptType.No, slice_size: Union[str, int] = DEFAULT_SLICE_SIZE, only_use_rapid_upload: bool = False, - task_id: Optional[TaskID] = None, - callback_for_monitor: Optional[Callable[[MultipartEncoderMonitor], None]] = None, + callback_for_monitor: Optional[Callable[[int], Any]] = None, + show_progress: bool = False, ) -> None: """Upload a file from `from_to[0]` to `from_to[1]` @@ -289,7 +289,8 @@ def upload_file( slice_size (Union[str, int], optional): Slice size. Defaults to DEFAULT_SLICE_SIZE. only_use_rapid_upload (bool, optional): Only use rapid upload. If rapid upload fails, raise exception. Defaults to False. task_id (Optional[TaskID], optional): Task ID. Defaults to None. - callback_for_monitor (Optional[Callable[[MultipartEncoderMonitor], None]], optional): Callback for progress monitor. Defaults to None. + callback_for_monitor (Optional[Callable[[int], Any]], optional): Callback for progress monitor. Defaults to None. + The callback should accept one argument which is the offset of the uploaded bytes. Examples: - Upload one file to one remote directory @@ -302,10 +303,25 @@ def upload_file( >>> localpath = "/local/file" >>> from_to = (localpath, remotedir) >>> upload_file(api, from_to) + ``` + + - With tqdm progress bar + + ```python + >>> from alipcs_py.alipcs import AliPCSApi + >>> from alipcs_py.commands.upload import upload, from_tos + >>> api = AliPCSApi(...) + >>> remotedir = "/remote/dir" + >>> localpath = "/local/file" + >>> from_to = (localpath, remotedir) + >>> with tqdm.tqdm(total=Path(localpath).stat().st_size) as pbar: + >>> upload_file(api, from_to, callback_for_monitor=lambda offset: pbar.n = offset) + ``` """ _wait_start() + # Upload basic info localpath, remotepath = from_to remotedir = posix_path_dirname(remotepath) @@ -318,12 +334,19 @@ def upload_file( filename = posix_path_basename(remotepath) + # Progress bar + task_id: Optional[TaskID] = None + if show_progress: + init_progress_bar() + task_id = _progress.add_task("upload", start=False, title=from_to[0]) + if not _need_to_upload(api, remotepath, check_name_mode): if task_id is not None: print(f"`{remotepath}` already exists.") remove_progress_task(task_id) return + # Upload IO info info = _init_encrypt_io(localpath, encrypt_password=encrypt_password, encrypt_type=encrypt_type) encrypt_io, encrypt_io_len, local_ctime, local_mtime = info if isinstance(slice_size, str): @@ -338,9 +361,16 @@ def upload_file( slice_completed = 0 - def callback_for_slice(monitor: MultipartEncoderMonitor): - if task_id is not None and progress_task_exists(task_id): - _progress.update(task_id, completed=slice_completed + monitor.bytes_read) + def callback_for_slice(offset: int): + if callback_for_monitor is not None: + callback_for_monitor(slice_completed + offset) + else: + if task_id is not None and progress_task_exists(task_id): + _progress.update(task_id, completed=slice_completed + offset) + + def teardown(): + encrypt_io.close() + remove_progress_task(task_id) # Rapid Upload try: @@ -377,14 +407,17 @@ def callback_for_slice(monitor: MultipartEncoderMonitor): task_id=task_id, ) if ok: + teardown() return except Exception as origin_err: + teardown() msg = f'Rapid upload "{localpath}" to "{remotepath}" failed. error: {origin_err}' logger.debug(msg) err = RapidUploadError(msg, localpath=localpath, remotepath=remotepath) raise err from origin_err if only_use_rapid_upload: + teardown() msg = f'Only use rapid upload but rapid upload failed. localpath: "{localpath}", remotepath: "{remotepath}"' logger.debug(msg) err = RapidUploadError(msg, localpath=localpath, remotepath=remotepath) @@ -440,11 +473,7 @@ def callback_for_slice(monitor: MultipartEncoderMonitor): ) upload_url = upload_urls[slice_idx] - api.upload_slice( - io, - upload_url, - callback=callback_for_slice if callback_for_monitor is None else callback_for_monitor, - ) + api.upload_slice(io, upload_url, callback_for_monitor=callback_for_slice) slice_idx += 1 break except Exception as origin_err: @@ -474,8 +503,6 @@ def callback_for_slice(monitor: MultipartEncoderMonitor): f"Hashs do not match between local file and remote file: local sha1 ({local_file_hash}) != remote sha1 ({remote_file_hash})" ) - remove_progress_task(task_id) - logger.debug( "`upload_file`: upload_slice and combine_slices success, task_id: %s", task_id, @@ -486,5 +513,4 @@ def callback_for_slice(monitor: MultipartEncoderMonitor): err = UploadError(msg, localpath=localpath, remotepath=remotepath) raise err from origin_err finally: - encrypt_io.close() - reset_progress_task(task_id) + teardown() From 9f1d3cad08d3c86300958742cf41cc0ea6fa4279 Mon Sep 17 00:00:00 2001 From: PeterDing Date: Tue, 9 Apr 2024 22:29:00 +0800 Subject: [PATCH 32/32] Update --- CHANGELOG.md | 318 +++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 35 ++---- 2 files changed, 325 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90f3064..fa7fc99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,323 @@ # Changelog +## v0.8.0 - 2024-04-09 + +Add new apis and remove unneeded apis. + +#### Inner datas + +1. **PcsFile** class + + - `path` + + Default is the name of the file. It will be different from different apis returned. See `AliPCSApi.meta`, `AliPCSApi.meta_by_path`, `AliPCSApi.get_file`, `AliPCSApi.list`, `AliPCSApi.list_iter`, `AliPCSApi.path_traceback`, `AliPCSApi.path`. + + - `update_download_url` + + The method is removed. Use `AliPCSApi.update_download_url` instead. + +2. **FromTo** type + + The original `FromTo` is a nametuple. We change it to a general type `FromTo = Tuple[F, T]` + +3. **PcsDownloadUrl** class + + - `expires` + + Add the method to check whether the `download_url` expires. + +#### Errors + +1. **AliPCSBaseError** class + + The base Exception class used for the PCS errors. + +2. **AliPCSError(AliPCSBaseError)** class + + The error returned from alipan server when the client’s request is incorrect or the token is expired. + + It throw at **AliPCS** class when an error occurs. + +3. **DownloadError(AliPCSBaseError)** class + + An error occurs when downloading action fails. + +4. **UploadError(AliPCSBaseError)** class + + An error occurs when uploading action fails. + +5. **RapidUploadError(UploadError)** class + + An error occurred when rapid uploading action fails. + +6. **make_alipcs_error** function + + Make an AliPCSError instance. + +7. **handle_error** function + + uses the `_error_max_retries` attribute of the wrapped method’s class to retry. + +#### Core APIs + +1. **AliPCS** class + + ```python + class AliPCS: + SHARE_AUTHS: Dict[str, SharedAuth] = {} + def __init__( + self, + refresh_token: str, + access_token: str = "", + token_type: str = "Bearer", + expire_time: int = 0, + user_id: str = "", + user_name: str = "", + nick_name: str = "", + device_id: str = "", + default_drive_id: str = "", + role: str = "", + status: str = "", + error_max_retries: int = 2, + max_keepalive_connections: int = 50, + max_connections: int = 50, + keepalive_expiry: float = 10 * 60, + connection_max_retries: int = 2, + ): ... + ``` + + The core alipan.com service apis. It directly handles the raw requests and responses of the service. + + **New/Changed APIs are following:** + + - `path_traceback` method (**New**) + + Traceback the path of the file by its file_id. Return the list of all parent directories' info from the file to the top level directory. + + - `meta_by_path` method (**New**) + + Get meta info of the file by its path. + + > Can not get the shared files' meta info. + + - `meta` method (**Changed**) + + Get meta info of the file by its file_id. + + - `exists` method (**Changed**) + + Check whether the file exists. Return True if the file exists and does not in the trash else False. + + - `exists_in_trash` method (**New**) + + Check whether the file exists in the trash. Return True if the file exists in the trash else False. + + - `walk` method (**New**) + + Walk through the directory tree by its file_id. + + - `download_link` method (**Changed**) + + Get download link of the file by its file_id. + + First try to get the download link from the meta info of the file. If the download link is not in the meta info, then request the getting download url api. + +2. **AliPCSApi** class + + ```python + class AliPCSApi: + def __init__( + self, + refresh_token: str, + access_token: str = "", + token_type: str = "", + expire_time: int = 0, + user_id: str = "", + user_name: str = "", + nick_name: str = "", + device_id: str = "", + default_drive_id: str = "", + role: str = "", + status: str = "", + error_max_retries: int = 2, + max_keepalive_connections: int = 50, + max_connections: int = 50, + keepalive_expiry: float = 10 * 60, + connection_max_retries: int = 2, + ): ... + ``` + + The core alipan.com service api with wrapped **AliPCS** class. It parses the raw content of response of AliPCS request into the inner data structions. + + - **New/Changed APIs are following:** + + - `path_traceback` method (**New**) + + Traceback the path of the file. Return the list of all `PcsFile`s from the file to the top level directory. + + > _Important_: + > The `path` property of the returned `PcsFile` has absolute path. + + - `meta_by_path` method (**New**) + + Get the meta of the the path. Can not get the shared files' meta info by their paths. + + > _Important_: + > The `path` property of the returned `PcsFile` is the argument `remotepath`. + + - `meta` method (**Changed**) + + Get meta info of the file. + + > _Important_: + > The `path` property of the returned `PcsFile` is only the name of the file. + + - `get_file` method (**New**) + + Get the file's info by the given `remotepath` or `file_id` + + If the `remotepath` is given, the `file_id` will be ignored. + + > _Important_: + > If the `remotepath` is given, the `path` property of the returned `PcsFile` is the `remotepath`. + > If the `file_id` is given, the `path` property of the returned `PcsFile` is only the name of the file. + + - `exists` method (**Changed**) + + Check whether the file exists. Return True if the file exists and does not in the trash else False. + + - `exists_in_trash` method (**Changed**) + + Check whether the file exists in the trash. Return True if the file exists in the trash else False. + + - `list` method (**Changed**) + + List files and directories in the given directory (which has the `file_id`). The return items size is limited by the `limit` parameter. If you want to list more, using the returned `next_marker` parameter for next `list` call. + + > _Important_: + > These PcsFile instances' path property is only the name of the file. + + - `list_iter` method (**Changed**) + + Iterate all files and directories at the directory (which has the `file_id`). + + > These returned PcsFile instances' path property is the path from the first sub-directory of the `file_id` to the file name. + > e.g. + > If the directory (owned `file_id`) has path `level0/`, a sub-directory which of path is + > `level0/level1/level2` then its corresponding PcsFile.path is `level1/level2`. + + - `path` method (**Changed**) + + Get the pcs file's info by the given absolute `remotepath` + + > _Important_: + > The `path` property of the returned `PcsFile` is the argument `remotepath`. + + - `list_path` method (**Removed**) + + - `list_path_iter` method (**Removed**) + + - `walk` method (**New**) + + Recursively Walk through the directory tree which has `file_id`. + + > _Important_: + > These PcsFile instances' path property is the path from the first sub-directory of the `file_id` to the file. + > e.g. + > If the directory (owned `file_id`) has path `level0/`, a sub-directory which of path is + > `level0/level1/level2` then its corresponding PcsFile.path is `level1/level2`. + + - `makedir` method (**Changed**) + + Make a directory in the `dir_id` directory + + > _Important_: + > The `path` property of the returned `PcsFile` is only the name of the directory. + + - **makedir_path** method (**Changed**) + + Make a directory by the absolute `remotedir` path + + Return the list of all `PcsFile`s from the directory to the top level directory. + + > _Important_: + > The `path` property of the returned `PcsFile` has absolute path. + + - `rename` method (**Changed**) + + Rename the file with `file_id` to `name` + + > _Important_: + > The `path` property of the returned `PcsFile` is only the name of the file. + + - `copy` method (**Changed**) + + Copy `file_ids[:-1]` to `file_ids[-1]` + + > _Important_: + > The `path` property of the returned `PcsFile` is only the name of the file. + + - `update_download_url` method (**New**) + + Update the download url of the `pcs_file` if it is expired. + + Return a new `PcsFile` with the updated download url. + +#### Download + +1. **MeDownloader** class + + ```python + class MeDownloader: + def __init__( + self, + range_request_io: RangeRequestIO, + localpath: PathType, + continue_: bool = False, + max_retries: int = 2, + done_callback: Optional[Callable[..., Any]] = None, + except_callback: Optional[Callable[[Exception], Any]] = None, + ) -> None: ... + ``` + +2. **download** module + + - `DownloadParams` class (**Removed**) + + We remove the `DownloadParams` instead of using arguments for function calling. + + - `download_file` function (**Changed**) + + `download_file` downloads one remote file to one local directory. Raise any error occurred. So giving the upper level caller to handle errors. + + - `download` function (**Changed**) + + `download` function downloads any number of remote files/directory to one local directory. It uses a `ThreadPoolExecutor` to download files concurrently and raise the exception if any error occurred. + +3. **upload** module + + - `UploadType` class (**Removed**) + + Alipan.com only support to upload a file through uploading slice parts one by one. + + So, the class is not needed. + + - `upload_file` function (**Changed**) + + Upload a file from one local file ( `from_to[0]`) to remote ( `from_to[1]`). + + First try to rapid upload, if failed, then upload file's slices. + + Raise exception if any error occurs. + + - `upload` function (**Changed**) + + Upload files in `from_to_list` to Alipan Drive. + + Use a `ThreadPoolExecutor` to upload files concurrently. + + Raise exception if any error occurs. + ## v0.7.0 - 2024-04-03 ### Updated diff --git a/README.md b/README.md index fdaee79..fb308c6 100644 --- a/README.md +++ b/README.md @@ -722,26 +722,6 @@ AliPCS-Py 首先会尝试秒传。如果秒传失败，会使用分片上传上 AliPCS-Py upload [OPTIONS] [LOCALPATHS]... REMOTEDIR ``` -指定上传方式： - -`--upload-type Many`: 同时上传多个文件。 - -适合大多数文件长度小于 100M 以下的情况。 - -``` -AliPCS-Py upload --upload-type Many [OPTIONS] [LOCALPATHS]... REMOTEDIR -``` - -`--upload-type One`: 一次只上传一个文件，但同时上传文件的多个分片。 - -适合大多数文件长度大于 1G 以上的情况。 - -**阿里网盘不支持单文件并发上传。`upload --upload-type One` 失效** - -``` -AliPCS-Py upload --upload-type One [OPTIONS] [LOCALPATHS]... REMOTEDIR -``` - 指定同时上传连接数量: `--max-workers` 默认为 CPU 核数。 @@ -758,14 +738,13 @@ AliPCS-Py upload --max-workers 4 [OPTIONS] [LOCALPATHS]... REMOTEDIR ### 选项 -| Option | Description | -| ---------------------------------------------------------- | --------------------------------------- | -| -t, --upload-type [One \| Many] | 上传方式，Many (默认): 同时上传多个文件 | -| --encrypt-password, --ep TEXT | 加密密码，默认使用用户设置的 | -| -e, --encrypt-type [No \| Simple \| ChaCha20 \| AES256CBC] | 文件加密方法，默认为 No 不加密 | -| -w, --max-workers INTEGER | 同时上传文件连接数量，默认为 CPU 核数 | -| --no-ignore-existing, --NI | 上传已经存在的文件 | -| --no-show-progress, --NP | 不显示上传进度 | +| Option | Description | +| ---------------------------------------------------------- | ------------------------------------- | +| --encrypt-password, --ep TEXT | 加密密码，默认使用用户设置的 | +| -e, --encrypt-type [No \| Simple \| ChaCha20 \| AES256CBC] | 文件加密方法，默认为 No 不加密 | +| -w, --max-workers INTEGER | 同时上传文件连接数量，默认为 CPU 核数 | +| --no-ignore-existing, --NI | 上传已经存在的文件 | +| --no-show-progress, --NP | 不显示上传进度 | ## 同步本地目录到远端