Skip to content

Commit

Permalink
enhance: e2e for go sdk
Browse files Browse the repository at this point in the history 
Signed-off-by: Yellow Shine <[email protected]>
  • Loading branch information
@yellow-shine
yellow-shine committed Aug 9, 2024
1 parent 1cff553 commit 91bb1b7
Show file tree
Hide file tree
Showing 2 changed files with 333 additions and 0 deletions.
239 changes: 239 additions & 0 deletions ci/jenkins/PR-for-go-sdk.groovy
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,239 @@
#!/usr/bin/env groovy

int total_timeout_minutes = 60 * 5
int e2e_timeout_seconds = 120 * 60
def imageTag = '123'
int case_timeout_seconds = 20 * 60
def chart_version = '4.1.8'
def release_name = ''
pipeline {
options {
timestamps()
timeout(time: total_timeout_minutes, unit: 'MINUTES')
buildDiscarder logRotator(artifactDaysToKeepStr: '30')
parallelsAlwaysFailFast()
preserveStashes(buildCount: 5)
disableConcurrentBuilds(abortPrevious: true)
}
agent {
kubernetes {
cloud '4am'
inheritFrom 'milvus-e2e-4am'
defaultContainer 'main'
yamlFile 'ci/jenkins/pod/rte-build.yaml'
customWorkspace '/home/jenkins/agent/milvus'
}
}
environment {
PROJECT_NAME = 'milvus'
SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
DOCKER_BUILDKIT = 1
ARTIFACTS = "${env.WORKSPACE}/_artifacts"
CI_DOCKER_CREDENTIAL_ID = 'harbor-milvus-io-registry'
MILVUS_HELM_NAMESPACE = 'milvus-ci'
DISABLE_KIND = true
HUB = 'harbor.milvus.io/milvus'
JENKINS_BUILD_ID = "${env.BUILD_ID}"
CI_MODE = 'pr'
SHOW_MILVUS_CONFIGMAP = true
}

stages {
stage('Build') {
steps {
container('main') {
dir('build') {
sh '''
MIRROR_URL="https://docker-nexus-ci.zilliz.cc" ./set_docker_mirror.sh
'''
}
dir('tests/scripts') {
script {
sh 'printenv'
def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim()
sh 'git config --global --add safe.directory /home/jenkins/agent/workspace'
def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
imageTag = "${env.BRANCH_NAME}-${date}-${gitShortCommit}"
withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]) {
sh """
TAG="${imageTag}" \
./e2e-k8s.sh \
--skip-export-logs \
--skip-install \
--skip-cleanup \
--skip-setup \
--skip-test
"""

// stash imageTag info for rebuild install & E2E Test only
sh "echo ${imageTag} > imageTag.txt"
stash includes: 'imageTag.txt', name: 'imageTag'
}
}
}
}
}
}

stage('Install & E2E Test') {
matrix {
axes {
axis {
name 'MILVUS_SERVER_TYPE'
values 'standalone', 'distributed'
}
axis {
name 'MILVUS_CLIENT'
values 'milvus-sdk-go'
}
}

stages {
stage('Install') {
steps {
container('main') {
stash includes: 'tests/**', name: 'testCode', useDefaultExcludes: false
stash includes: 'client/**', name: 'clientCode', useDefaultExcludes: false
dir('tests/scripts') {
script {
sh 'printenv'
def clusterEnabled = 'false'
def valuesFile = 'pr-4am.yaml'
if ("${MILVUS_SERVER_TYPE}".contains('distributed')) {
clusterEnabled = 'true'
}
if ("${imageTag}" == '') {
dir('imageTag') {
try {
unstash 'imageTag'
imageTag = sh(returnStdout: true, script: 'cat imageTag.txt | tr -d \'\n\r\'')
}catch (e) {
print 'No Image Tag info remained ,please rerun build to build new image.'
exit 1
}
}
}
// modify values file to enable kafka
if ("${MILVUS_SERVER_TYPE}".contains('kafka')) {
sh '''
apt-get update
apt-get install wget -y
wget https://github.com/mikefarah/yq/releases/download/v4.34.1/yq_linux_amd64 -O /usr/bin/yq
chmod +x /usr/bin/yq
'''
sh """
cp values/ci/pr-4am.yaml values/ci/pr_kafka.yaml
yq -i '.pulsar.enabled=false' values/ci/pr_kafka.yaml
yq -i '.kafka.enabled=true' values/ci/pr_kafka.yaml
yq -i '.kafka.metrics.kafka.enabled=true' values/ci/pr_kafka.yaml
yq -i '.kafka.metrics.jmx.enabled=true' values/ci/pr_kafka.yaml
yq -i '.kafka.metrics.serviceMonitor.enabled=true' values/ci/pr_kafka.yaml
"""
}
withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]) {
sh """
MILVUS_CLUSTER_ENABLED=${clusterEnabled} \
MILVUS_HELM_REPO="https://nexus-ci.zilliz.cc/repository/milvus-proxy" \
TAG=${imageTag}\
./e2e-k8s.sh \
--skip-export-logs \
--skip-cleanup \
--skip-setup \
--skip-test \
--skip-build \
--skip-build-image \
--install-extra-arg "
--set etcd.metrics.enabled=true \
--set etcd.metrics.podMonitor.enabled=true \
--set indexCoordinator.gc.interval=1 \
--set indexNode.disk.enabled=true \
--set queryNode.disk.enabled=true \
--set standalone.disk.enabled=true \
--version ${chart_version} \
-f values/ci/${valuesFile}"
"""
}

release_name = sh(returnStdout: true, script: './get_release_name.sh').trim()
}
}
}
}
}
stage('E2E Test') {
options {
skipDefaultCheckout()
}
agent {
kubernetes {
cloud '4am'
inheritFrom 'default'
defaultContainer 'main'
yamlFile 'ci/jenkins/pod/e2e-go-sdk.yaml'
customWorkspace '/home/jenkins/agent/milvus'
}
}
steps {
container('gotestsum') {
unstash('testCode')
unstash('clientCode')
dir('tests/scripts') {
script {
def clusterEnabled = 'false'
if ("${MILVUS_SERVER_TYPE}".contains('distributed')) {
clusterEnabled = 'true'
}
}
}

dir('tests/go_client') {

script {
sh """
gotestsum --format testname --hide-summary=output ./testcases/... --tags L0 --addr=${release_name}-milvus.milvus-ci:19530 -timeout=60m
"""

}

}
}
}
}
}
post {
always {
container('main') {
dir('tests/scripts') {
script {
sh "kubectl get pods -n ${MILVUS_HELM_NAMESPACE} | grep ${release_name} "
sh "./uninstall_milvus.sh --release-name ${release_name}"
sh "./ci_logs.sh --log-dir /ci-logs --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \
--release-name ${release_name}"
dir("${env.ARTIFACTS}") {
archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz", allowEmptyArchive: true
}
}
}
}
}
}
}
}
}
post {
unsuccessful {
container('jnlp') {
dir('tests/scripts') {
script {
def authorEmail = sh(returnStdout: true, script: './get_author_email.sh ')
emailext subject: '$DEFAULT_SUBJECT',
body: '$DEFAULT_CONTENT',
recipientProviders: [developers(), culprits()],
replyTo: '$DEFAULT_REPLYTO',
to: "${authorEmail},[email protected]"
}
}
}
}
}
}
94 changes: 94 additions & 0 deletions ci/jenkins/pod/e2e-go-sdk.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus-e2e
task: milvus-e2e-pytest
namespace: milvus-ci
spec:
enableServiceLinks: false
containers:
- name: gotestsum
image: harbor.milvus.io/dockerhub/milvusdb/gotestsum:20240807
resources:
limits:
cpu: "6"
memory: 12Gi
requests:
cpu: "0.5"
memory: 5Gi
# command: ["sh", "-c", "sleep infinity"]
args: ["cat"]
tty: true
volumeMounts:
- mountPath: /ci-logs
name: ci-logs
- mountPath: /go/pkg/mod
name: go-mod
- name: main
image: milvusdb/krte:20211213-dcc15e9
env:
- name: DOCKER_IN_DOCKER_ENABLED
value: "true"
- name: DOCKER_VOLUME_DIRECTORY
value: "/mnt/disk/.docker"
- name: DOCKER_CLIENT_TIMEOUT
value: 240
- name: COMPOSE_HTTP_TIMEOUT
value: 240
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
tty: true
securityContext:
privileged: true
args: ["cat"]
resources:
limits:
cpu: "6"
memory: 12Gi
requests:
cpu: "0.5"
memory: 2Gi
volumeMounts:
- mountPath: /docker-graph
name: docker-graph
- mountPath: /var/lib/docker
name: docker-root
- mountPath: /lib/modules
name: modules
readOnly: true
- mountPath: /sys/fs/cgroup
name: cgroup
- mountPath: /mnt/disk/.docker
name: build-cache
subPath: docker-volume
- mountPath: /ci-logs
name: ci-logs
volumes:
- emptyDir: {}
name: docker-graph
- emptyDir: {}
name: docker-root
- hostPath:
path: /tmp/krte/cache
type: DirectoryOrCreate
name: build-cache
- hostPath:
path: /tmp/krte/cache/docker-volume/amd64-ubuntu22.04-go-mod
type: DirectoryOrCreate
name: go-mod
- hostPath:
path: /lib/modules
type: Directory
name: modules
- hostPath:
path: /sys/fs/cgroup
type: Directory
name: cgroup
- name: ci-logs
nfs:
path: /volume1/ci-logs
# path: /volume1/4am-logs
server: 172.16.70.249

0 comments on commit 91bb1b7

Please sign in to comment.