using pipeline 1 and batch size 1 on CE upload

redis-performance · Oct 2, 2024 · 6f50710 · 6f50710
1 parent 2d8e250
commit 6f50710
Show file tree

Hide file tree

Showing 2 changed files with 71 additions and 16 deletions.
diff --git a/experiments/configurations/create-intel-ce-edition.py b/experiments/configurations/create-intel-ce-edition.py
@@ -0,0 +1,39 @@
+import json
+
+ms = [4, 8, 16, 32]
+ef_constructs = [4, 8, 16, 32]
+ef_runtimes = [4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192]
+data_type = "FLOAT16"
+for algo in ["hnsw"]:
+    configs = []
+    for m in ms:
+        for ef_construct in ef_constructs:
+            config = {
+                "name": f"redis-intel-{data_type.lower()}-{algo}-m-{m}-ef-{ef_construct}",
+                "engine": "redis",
+                "connection_params": {},
+                "collection_params": {
+                    "algorithm": algo,
+                    "data_type": data_type,
+                    f"{algo}_config": {"M": m, "EF_CONSTRUCTION": ef_construct},
+                },
+                "search_params": [],
+                "upload_params": {
+                    "parallel": 1,
+                    "batch_size": 1,
+                    "algorithm": algo,
+                    "data_type": data_type,
+                },
+            }
+            for client in [1, 50, 100, 200]:
+                for ef_runtime in ef_runtimes:
+                    test_config = {
+                        "parallel": client,
+                        "search_params": {"ef": ef_runtime, "data_type": data_type},
+                    }
+                    config["search_params"].append(test_config)
+            configs.append(config)
+    fname = f"redis-intel-{algo}-single-node.json"
+    with open(fname, "w") as json_fd:
+        json.dump(configs, json_fd, indent=2)
+        print(f"created {len(configs)} configs for {fname}.")
diff --git a/experiments/configurations/redis-intel-hnsw-single-node.json b/experiments/configurations/redis-intel-hnsw-single-node.json
@@ -350,7 +350,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -706,7 +707,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -1062,7 +1064,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -1418,7 +1421,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -1774,7 +1778,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -2130,7 +2135,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -2486,7 +2492,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -2842,7 +2849,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -3198,7 +3206,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -3554,7 +3563,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -3910,7 +3920,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -4266,7 +4277,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -4622,7 +4634,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -4978,7 +4991,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -5334,7 +5348,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }
@@ -5690,7 +5705,8 @@
       }
     ],
     "upload_params": {
-      "parallel": 128,
+      "parallel": 1,
+      "batch_size": 1,
       "algorithm": "hnsw",
       "data_type": "FLOAT16"
     }