Update ci docker image and optimize gpu build (#395)

Signed-off-by: Yudong Cai <[email protected]>
zilliztech · Feb 21, 2024 · b4149ed · b4149ed
1 parent c02e01b
commit b4149ed
Show file tree

Hide file tree

Showing 6 changed files with 18 additions and 9 deletions.
diff --git a/ci/E2E_GPU.groovy b/ci/E2E_GPU.groovy
@@ -1,4 +1,4 @@
-int total_timeout_minutes = 120
+int total_timeout_minutes = 240
 def knowhere_wheel=''
 pipeline {
     agent {
@@ -33,6 +33,8 @@ pipeline {
                         sh "pip3 install conan==1.61.0"
                         sh "conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local"
                         sh "cmake --version"
+                        sh "nvidia-smi --query-gpu=name --format=csv,noheader"
+                        sh "./scripts/prepare_gpu_build.sh"
                         sh "mkdir build"
                         sh "cd build/ && conan install .. --build=missing -o with_diskann=True -o with_raft=True -s compiler.libcxx=libstdc++11 && conan build .."
                         sh "cd python && VERSION=${version} python3 setup.py bdist_wheel"

diff --git a/ci/UT_GPU.groovy b/ci/UT_GPU.groovy
@@ -1,4 +1,4 @@
-int total_timeout_minutes = 120
+int total_timeout_minutes = 240
 def knowhere_wheel=''
 pipeline {
     agent {
@@ -29,6 +29,8 @@ pipeline {
                         sh "pip3 install conan==1.61.0"
                         sh "conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local"
                         sh "cmake --version"
+                        sh "nvidia-smi --query-gpu=name --format=csv,noheader"
+                        sh "./scripts/prepare_gpu_build.sh"
                         sh "mkdir build"
                         sh "cd build/ && conan install .. --build=missing -s build_type=Release -o with_diskann=True -o with_ut=True -o with_raft=True -s compiler.libcxx=libstdc++11 \
                               && conan build .. \

diff --git a/ci/pod/e2e-cpu.yaml b/ci/pod/e2e-cpu.yaml
@@ -24,20 +24,18 @@ spec:
           topologyKey: "kubernetes.io/hostname"
   containers:
   - name: build
-    image: milvusdb/knowhere-gpu-build:amd64-ubuntu20.04-20231117-7a169d0
+    image: milvusdb/knowhere-cpu-build:amd64-ubuntu20.04-20240220-c02e01b
     tty: true
     args: ["cat"]
   - name: main
-    image: milvusdb/knowhere-gpu-e2e:amd64-ubuntu20.04-20221011-1249a57
+    image: milvusdb/knowhere-cpu-build:amd64-ubuntu20.04-20240220-c02e01b
     tty: true
     args: ["cat"]
     resources:
       requests:
-        nvidia.com/gpu: 1
         memory: "30Gi"
         cpu: "8"
       limits:
-        nvidia.com/gpu: 1
         memory: "30Gi"
         cpu: "8"
     volumeMounts:

diff --git a/ci/pod/e2e-gpu.yaml b/ci/pod/e2e-gpu.yaml
@@ -24,11 +24,11 @@ spec:
           topologyKey: "kubernetes.io/hostname"
   containers:
   - name: build
-    image: milvusdb/knowhere-gpu-build:amd64-ubuntu20.04-20231117-7a169d0
+    image: milvusdb/knowhere-gpu-build:amd64-ubuntu20.04-20240117-5230b6a
     tty: true
     args: ["cat"]
   - name: main
-    image: nvidia/cuda:11.6.0-runtime-ubuntu20.04
+    image: milvusdb/knowhere-gpu-build:amd64-ubuntu20.04-20240117-5230b6a
     tty: true
     args: ["cat"]
     resources:

diff --git a/ci/pod/ut-gpu.yaml b/ci/pod/ut-gpu.yaml
@@ -24,7 +24,7 @@ spec:
           topologyKey: "kubernetes.io/hostname"
   containers:
   - name: build
-    image: milvusdb/knowhere-gpu-build:amd64-ubuntu20.04-20231117-7a169d0
+    image: milvusdb/knowhere-gpu-build:amd64-ubuntu20.04-20240117-5230b6a
     env:
       - name: ASAN_OPTIONS
         value: protect_shadow_gap=0

diff --git a/scripts/prepare_gpu_build.sh b/scripts/prepare_gpu_build.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# This script is to modify CMakeLists.txt to build knowhere for certain GPU serials
+#   GTX 1060 - sm_61
+#   GTX 1660 - sm_75
+sed 's/set(CMAKE_CUDA_ARCHITECTURES \${supported_archs})/set(CMAKE_CUDA_ARCHITECTURES \"61-real;75-real\")/' CMakeLists.txt > tmp
+mv tmp CMakeLists.txt