add sizing threshold detection for streaming instead of batch (#426)

NVIDIA · Feb 12, 2025 · 3c68091 · 3c68091
1 parent bfcc946
commit 3c68091
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/client/src/nv_ingest_client/util/milvus.py b/client/src/nv_ingest_client/util/milvus.py
@@ -593,6 +593,7 @@ def write_to_nvingest_collection(
     access_key: str = "minioadmin",
     secret_key: str = "minioadmin",
     bucket_name: str = "a-bucket",
+    threshold: int = 10,
 ):
     """
     This function takes the input records and creates a corpus,
@@ -655,6 +656,8 @@ def write_to_nvingest_collection(
         bm25_ef.load(bm25_save_path)
     client = MilvusClient(milvus_uri)
     schema = Collection(collection_name).schema
+    if len(records) < threshold:
+        stream = True
     if stream:
         stream_insert_milvus(
             records,