diff --git a/experiments/configurations/qdrant-single-node-bq-rps.json b/experiments/configurations/qdrant-single-node-bq-rps.json new file mode 100644 index 00000000..09f95c83 --- /dev/null +++ b/experiments/configurations/qdrant-single-node-bq-rps.json @@ -0,0 +1,284 @@ +[ + { + "name": "qdrant-bq-rps-default", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "quantization_config": { "binary": {"always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } } + ], + "upload_params": { "parallel": 1, "batch_size": 1024 } + }, + { + "name": "qdrant-bq-rps-m-16-ef-128", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { "binary": {"always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-bq-rps-m-32-ef-128", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 32, "ef_construct": 128 }, + "quantization_config": { "binary": {"always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-bq-rps-m-32-ef-256", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 32, "ef_construct": 256 }, + "quantization_config": { "binary": {"always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-bq-rps-m-32-ef-512", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 32, "ef_construct": 512 }, + "quantization_config": { "binary": {"always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-bq-rps-m-64-ef-256", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 64, "ef_construct": 256 }, + "quantization_config": { "binary": {"always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-bq-rps-m-64-ef-512", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 64, "ef_construct": 512 }, + "quantization_config": { "binary": {"always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + } +] diff --git a/experiments/configurations/qdrant-single-node-sq-rps.json b/experiments/configurations/qdrant-single-node-sq-rps.json new file mode 100644 index 00000000..8bb83d62 --- /dev/null +++ b/experiments/configurations/qdrant-single-node-sq-rps.json @@ -0,0 +1,284 @@ +[ + { + "name": "qdrant-sq-rps-default", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "quantization_config": { "scalar": {"type": "int8", "quantile": 0.99, "always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } } + ], + "upload_params": { "parallel": 1, "batch_size": 1024 } + }, + { + "name": "qdrant-sq-rps-m-16-ef-128", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 16, "ef_construct": 128 }, + "quantization_config": { "scalar": {"type": "int8", "quantile": 0.99, "always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-sq-rps-m-32-ef-128", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 32, "ef_construct": 128 }, + "quantization_config": { "scalar": {"type": "int8", "quantile": 0.99, "always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-sq-rps-m-32-ef-256", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 32, "ef_construct": 256 }, + "quantization_config": { "scalar": {"type": "int8", "quantile": 0.99, "always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-sq-rps-m-32-ef-512", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 32, "ef_construct": 512 }, + "quantization_config": { "scalar": {"type": "int8", "quantile": 0.99, "always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-sq-rps-m-64-ef-256", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 64, "ef_construct": 256 }, + "quantization_config": { "scalar": {"type": "int8", "quantile": 0.99, "always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + }, + { + "name": "qdrant-sq-rps-m-64-ef-512", + "engine": "qdrant", + "connection_params": { "timeout": 30 }, + "collection_params": { + "optimizers_config": { "max_segment_size": 1000000, "memmap_threshold":10000000, "default_segment_number":2 }, + "hnsw_config": { "m": 64, "ef_construct": 512 }, + "quantization_config": { "scalar": {"type": "int8", "quantile": 0.99, "always_ram": true} } + }, + "search_params": [ + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 1, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 64, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 128, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 256, "quantization": { "rescore": true, "oversampling": 8.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 1.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 2.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 4.0 } } }, + { "parallel": 100, "search_params": { "hnsw_ef": 512, "quantization": { "rescore": true, "oversampling": 8.0 } } } + ], + "upload_params": { "parallel": 16, "batch_size": 1024 } + } +] diff --git a/tools/remote/setup_benchmark_client.sh b/tools/remote/setup_benchmark_client.sh index 17d16453..41c36f18 100644 --- a/tools/remote/setup_benchmark_client.sh +++ b/tools/remote/setup_benchmark_client.sh @@ -19,7 +19,7 @@ python3 -m poetry install # if using qdrant vector db if [ "$VECTOR_DB" == "qdrant" ]; then - QDRANT_CONFIGS=$(cat experiments/configurations/qdrant-single-node{-rps,}.json | jq '.[] | .name' | grep -E 'qdrant(-rps)?-m-.*-ef-.*' | sed 's/"//g') + QDRANT_CONFIGS=$(cat experiments/configurations/qdrant-single-node{-bq-rps,-sq-rps,-rps,}.json | jq '.[] | .name' | grep -E 'qdrant(-(rps|bq-rps|sq-rps))?-m-.*-ef-.*' | sed 's/"//g') for QDRANT_CONFIG in $QDRANT_CONFIGS; do # upload