fix(tqdm): import tqdm to support jupyter

src/datachain/cache.py

@@ -9,8 +9,6 @@
 from dvc_objects.fs.utils import remove
 from fsspec.callbacks import Callback, TqdmCallback
 
-from .progress import Tqdm
-
 if TYPE_CHECKING:
     from datachain.client import Client
     from datachain.lib.file import File
@@ -86,9 +84,11 @@ async def download(
         size = file.size
         if size < 0:
             size = await client.get_size(from_path, version_id=file.version)
+        from tqdm.auto import tqdm
+
         cb = callback or TqdmCallback(
             tqdm_kwargs={"desc": odb_fs.name(from_path), "bytes": True, "leave": False},
-            tqdm_cls=Tqdm,
+            tqdm_cls=tqdm,
             size=size,
         )
         try:

src/datachain/catalog/catalog.py

@@ -28,7 +28,7 @@
 import sqlalchemy as sa
 import yaml
 from sqlalchemy import Column
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.cache import DataChainCache
 from datachain.client import Client

src/datachain/client/azure.py

@@ -2,7 +2,7 @@
 from urllib.parse import parse_qs, urlsplit, urlunsplit
 
 from adlfs import AzureBlobFileSystem
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.lib.file import File
 

src/datachain/client/fsspec.py

@@ -23,7 +23,7 @@
 from dvc_objects.fs.system import reflink
 from fsspec.asyn import get_loop, sync
 from fsspec.callbacks import DEFAULT_CALLBACK, Callback
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.cache import DataChainCache
 from datachain.client.fileslice import FileWrapper

src/datachain/client/gcs.py

@@ -7,7 +7,7 @@
 
 from dateutil.parser import isoparse
 from gcsfs import GCSFileSystem
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.lib.file import File
 

src/datachain/client/s3.py

@@ -4,7 +4,7 @@
 
 from botocore.exceptions import NoCredentialsError
 from s3fs import S3FileSystem
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.lib.file import File
 

src/datachain/data_storage/sqlite.py

@@ -21,7 +21,7 @@
 from sqlalchemy.sql import func
 from sqlalchemy.sql.expression import bindparam, cast
 from sqlalchemy.sql.selectable import Select
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 import datachain.sql.sqlite
 from datachain.data_storage import AbstractDBMetastore, AbstractWarehouse

src/datachain/data_storage/warehouse.py

@@ -14,7 +14,7 @@
 from sqlalchemy import Table, case, select
 from sqlalchemy.sql import func
 from sqlalchemy.sql.expression import true
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.client import Client
 from datachain.data_storage.schema import convert_rows_custom_column_types

src/datachain/lib/arrow.py

@@ -7,7 +7,7 @@
 import pyarrow as pa
 from fsspec.core import split_protocol
 from pyarrow.dataset import CsvFileFormat, dataset
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.lib.data_model import dict_to_data_model
 from datachain.lib.file import ArrowRow, File

src/datachain/lib/hf.py

@@ -29,7 +29,7 @@
 from typing import TYPE_CHECKING, Any, Union
 
 import PIL
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.lib.arrow import arrow_type_mapper
 from datachain.lib.data_model import DataModel, DataType, dict_to_data_model

src/datachain/listing.py

@@ -7,7 +7,7 @@
 
 from sqlalchemy import Column
 from sqlalchemy.sql import func
-from tqdm import tqdm
+from tqdm.auto import tqdm
 
 from datachain.node import DirType, Node, NodeWithPath
 from datachain.sql.functions import path as pathfunc

src/datachain/progress.py

@@ -7,15 +7,16 @@
 
 from fsspec import Callback
 from fsspec.callbacks import TqdmCallback
-from tqdm import tqdm
+from tqdm.auto import tqdm
+from tqdm.auto import tqdm as ntqdm
 
 from datachain.utils import env2bool
 
 logger = logging.getLogger(__name__)
 tqdm.set_lock(RLock())
 
 
-class Tqdm(tqdm):
+class Tqdm(ntqdm):
     """
     maximum-compatibility tqdm-based progressbars
     """
@@ -148,6 +149,8 @@ def __init__(self, tqdm_kwargs=None, *args, **kwargs):
         self.files_count = 0
         tqdm_kwargs = tqdm_kwargs or {}
         tqdm_kwargs.setdefault("postfix", {}).setdefault("files", self.files_count)
+        kwargs = kwargs or {}
+        kwargs["tqdm_cls"] = ntqdm
         super().__init__(tqdm_kwargs, *args, **kwargs)
 
     def increment_file_count(self, n: int = 1) -> None:

src/datachain/query/dataset.py

@@ -33,6 +33,7 @@
 from sqlalchemy.sql.expression import label
 from sqlalchemy.sql.schema import TableClause
 from sqlalchemy.sql.selectable import Select
+from tqdm.auto import tqdm
 
 from datachain.asyn import ASYNC_WORKERS, AsyncMapper, OrderedMapper
 from datachain.catalog.catalog import clone_catalog_with_cache
@@ -366,12 +367,16 @@ def get_download_callback(suffix: str = "", **kwargs) -> CombinedDownloadCallbac
 
 
 def get_processed_callback() -> Callback:
-    return TqdmCallback({"desc": "Processed", "unit": " rows", "leave": False})
+    return TqdmCallback(
+        {"desc": "Processed", "unit": " rows", "leave": False}, tqdm_cls=tqdm
+    )
 
 
 def get_generated_callback(is_generator: bool = False) -> Callback:
     if is_generator:
-        return TqdmCallback({"desc": "Generated", "unit": " rows", "leave": False})
+        return TqdmCallback(
+            {"desc": "Generated", "unit": " rows", "leave": False}, tqdm_cls=tqdm
+        )
     return DEFAULT_CALLBACK