forked from Kaggle/docker-python
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Jenkinsfile
130 lines (116 loc) · 3.88 KB
/
Jenkinsfile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
String cron_string = BRANCH_NAME == "master" ? "H 12 * * 1,3,5" : ""
pipeline {
agent { label 'ephemeral-linux' }
options {
// The Build GPU stage depends on the image from the Push CPU stage
disableConcurrentBuilds()
}
triggers {
cron(cron_string)
}
environment {
GIT_COMMIT_SHORT = sh(returnStdout: true, script:"git rev-parse --short=7 HEAD").trim()
GIT_COMMIT_SUBJECT = sh(returnStdout: true, script:"git log --format=%s -n 1 HEAD").trim()
GIT_COMMIT_AUTHOR = sh(returnStdout: true, script:"git log --format='%an' -n 1 HEAD").trim()
GIT_COMMIT_SUMMARY = "`<https://github.com/Kaggle/docker-python/commit/${GIT_COMMIT}|${GIT_COMMIT_SHORT}>` ${GIT_COMMIT_SUBJECT} - ${GIT_COMMIT_AUTHOR}"
SLACK_CHANNEL = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"#kernelops\"; else echo \"#builds\"; fi").trim()
PRETEST_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"ci-pretest\"; else echo \"${GIT_BRANCH}-pretest\"; fi").trim()
STAGING_TAG = sh(returnStdout: true, script: "if [[ \"${GIT_BRANCH}\" == \"master\" ]]; then echo \"staging\"; else echo \"${GIT_BRANCH}-staging\"; fi").trim()
}
stages {
stage('Docker CPU Build') {
steps {
sh '''#!/bin/bash
set -exo pipefail
./build | ts
./push ${PRETEST_TAG}
'''
}
}
stage('Test CPU Image') {
steps {
sh '''#!/bin/bash
set -exo pipefail
date
./test --image gcr.io/kaggle-images/python:${PRETEST_TAG}
'''
}
}
stage('Push CPU Image') {
steps {
sh '''#!/bin/bash
set -exo pipefail
date
./push ${STAGING_TAG}
'''
}
}
stage('Docker GPU Build') {
// A GPU is not required to build this image. However, in our current setup,
// the default runtime is set to nvidia (as opposed to runc) and there
// is no option to specify a runtime for the `docker build` command.
//
// TODO(rosbo) don't set `nvidia` as the default runtime and use the
// `--runtime=nvidia` flag for the `docker run` command when GPU support is needed.
agent { label 'ephemeral-linux-gpu' }
steps {
sh '''#!/bin/bash
set -exo pipefail
docker image prune -f # remove previously built image to prevent disk from filling up
./build --gpu --base-image-tag ${STAGING_TAG} | ts
./push --gpu ${PRETEST_TAG}
'''
}
}
stage('Test GPU Image') {
agent { label 'ephemeral-linux-gpu' }
steps {
sh '''#!/bin/bash
set -exo pipefail
date
./test --gpu --image gcr.io/kaggle-private-byod/python:${PRETEST_TAG}
'''
}
}
stage('Push GPU Image') {
agent { label 'ephemeral-linux-gpu' }
steps {
sh '''#!/bin/bash
set -exo pipefail
date
./push --gpu ${STAGING_TAG}
'''
}
}
stage('Package Versions') {
parallel {
stage('CPU Diff') {
steps {
sh '''#!/bin/bash
./diff
'''
}
}
stage('GPU Diff') {
agent { label 'ephemeral-linux-gpu' }
steps {
sh '''#!/bin/bash
./diff --gpu
'''
}
}
}
}
}
post {
failure {
slackSend color: 'danger', message: "*<${env.BUILD_URL}console|${JOB_NAME} failed>* ${GIT_COMMIT_SUMMARY} @kernels-backend-ops", channel: env.SLACK_CHANNEL
}
success {
slackSend color: 'good', message: "*<${env.BUILD_URL}console|${JOB_NAME} passed>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
}
aborted {
slackSend color: 'warning', message: "*<${env.BUILD_URL}console|${JOB_NAME} aborted>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
}
}
}