Skip to content

Commit

Permalink
Fix spacing
Browse files Browse the repository at this point in the history
  • Loading branch information
@roceb
roceb committed Jan 29, 2025
1 parent 9b568ab commit b90fe10
Show file tree
Hide file tree
Showing 6 changed files with 136 additions and 137 deletions.
2 changes: 1 addition & 1 deletion etc/scipipe/build_matrix.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ versiondb:
release_tag_org: lsst
lsstsw:
github_repo: lsst/lsstsw
git_ref: tickets/DM-46554
git_ref: main
ciscripts:
github_repo: lsst-sqre/ci-scripts
git_ref: main
Expand Down
16 changes: 9 additions & 7 deletions pipelines/lib/util.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -686,16 +686,18 @@ def void createDirs(List eds) {
* @param filename String Output filename.
*/
def void getManifest(String rebuildId, String filename) {
def buildJob = 'release/run-rebuild'
def manifest_artifact = 'lsstsw/build/manifest.txt'
def buildJob = 'release/run-rebuild'

step([$class: 'CopyArtifact',
projectName: buildJob,
filter: manifest_artifact,
selector: [
$class: 'SpecificBuildSelector',
filter: manifest_artifact,
selector: [
$class: 'SpecificBuildSelector',
buildNumber: rebuildId // wants a string
],
])
],
])

def manifest = readFile manifest_artifact
writeFile(file: filename, text: manifest)
} // getManifest
Expand Down Expand Up @@ -1383,6 +1385,7 @@ def String runRebuild(Map p) {
parameters: jobParameters,
wait: true,
)

nodeTiny {
manifestArtifact = 'lsstsw/build/manifest.txt'

Expand Down Expand Up @@ -2301,5 +2304,4 @@ def void nodeWrap(String label, Closure run) {
}
}


return this;
183 changes: 89 additions & 94 deletions pipelines/release/docker/build_stack.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,85 +75,85 @@ notify.wrap {
def slug = util.lsstswConfigSlug(lsstswConfig)
matrix[slug] ={

def newinstallImage = newinstall.docker_registry.repo
def newinstallTagBase = newinstall.docker_registry.tag
def splenvRef = lsstswConfig.splenv_ref
if (params.SPLENV_REF) {
splenvRef = params.SPLENV_REF
}
def newinstallImage = newinstall.docker_registry.repo
def newinstallTagBase = newinstall.docker_registry.tag
def splenvRef = lsstswConfig.splenv_ref
if (params.SPLENV_REF) {
splenvRef = params.SPLENV_REF
}

def baseImage = "${newinstallImage}:${newinstallTagBase}-${splenvRef}"
def baseImage = "${newinstallImage}:${newinstallTagBase}-${splenvRef}"

def image = null
def repo = null
def image = null
def repo = null


def run = {
stage('checkout') {
repo = git([
url: githubRepo,
branch: gitRef,
])
}
def run = {
stage('checkout') {
repo = git([
url: githubRepo,
branch: gitRef,
])
}

stage('build') {
def opt = []
// ensure base image is always up to date
opt << '--pull=true'
opt << '--no-cache'
opt << "--build-arg EUPS_PRODUCTS=\"${products}\""
opt << "--build-arg EUPS_TAG=\"${eupsTag}\""
opt << "--build-arg DOCKERFILE_GIT_BRANCH=\"${repo.GIT_BRANCH}\""
opt << "--build-arg DOCKERFILE_GIT_COMMIT=\"${repo.GIT_COMMIT}\""
opt << "--build-arg DOCKERFILE_GIT_URL=\"${repo.GIT_URL}\""
opt << "--build-arg JENKINS_JOB_NAME=\"${env.JOB_NAME}\""
opt << "--build-arg JENKINS_BUILD_ID=\"${env.BUILD_ID}\""
opt << "--build-arg JENKINS_BUILD_URL=\"${env.RUN_DISPLAY_URL}\""
opt << "--build-arg BASE_IMAGE=\"${baseImage}\""
opt << "--build-arg SHEBANGTRON_URL=\"${shebangtronUrl}\""
opt << "--build-arg VERSIONDB_MANIFEST_ID=\"${manifestId}\""
opt << "--build-arg LSST_COMPILER=\"${lsstCompiler}\""
opt << "--build-arg LSST_SPLENV_REF=\"${splenvRef}\""
opt << "--load"
opt << '.'

dir(buildDir) {
image = docker.build("${dockerRepo}", opt.join(' '))
image2 = docker.build("panda-dev-1a74/${dockerRepo}", opt.join(' '))
image3 = docker.build("lsstsqre/almalinux", opt.join(' '))
stage('build') {
def opt = []
// ensure base image is always up to date
opt << '--pull=true'
opt << '--no-cache'
opt << "--build-arg EUPS_PRODUCTS=\"${products}\""
opt << "--build-arg EUPS_TAG=\"${eupsTag}\""
opt << "--build-arg DOCKERFILE_GIT_BRANCH=\"${repo.GIT_BRANCH}\""
opt << "--build-arg DOCKERFILE_GIT_COMMIT=\"${repo.GIT_COMMIT}\""
opt << "--build-arg DOCKERFILE_GIT_URL=\"${repo.GIT_URL}\""
opt << "--build-arg JENKINS_JOB_NAME=\"${env.JOB_NAME}\""
opt << "--build-arg JENKINS_BUILD_ID=\"${env.BUILD_ID}\""
opt << "--build-arg JENKINS_BUILD_URL=\"${env.RUN_DISPLAY_URL}\""
opt << "--build-arg BASE_IMAGE=\"${baseImage}\""
opt << "--build-arg SHEBANGTRON_URL=\"${shebangtronUrl}\""
opt << "--build-arg VERSIONDB_MANIFEST_ID=\"${manifestId}\""
opt << "--build-arg LSST_COMPILER=\"${lsstCompiler}\""
opt << "--build-arg LSST_SPLENV_REF=\"${splenvRef}\""
opt << "--load"
opt << '.'

dir(buildDir) {
image = docker.build("${dockerRepo}", opt.join(' '))
image2 = docker.build("panda-dev-1a74/${dockerRepo}", opt.join(' '))
image3 = docker.build("lsstsqre/almalinux", opt.join(' '))
}
}
}
stage('push') {
def digest = null
if (!noPush) {
docker.withRegistry(
'https://index.docker.io/v1/',
'dockerhub-sqreadmin'
) {
registryTags.each { name ->
image.push(name)
stage('push') {
def digest = null
def arch = lsstswConfig.tokenize('-').last()
if (!noPush) {
docker.withRegistry(
'https://index.docker.io/v1/',
'dockerhub-sqreadmin'
) {
registryTags.each { name ->
image.push(name+arch)
}
newRegistryTags.each { name ->
image3.push(name+arch)
}
}
newRegistryTags.each { name ->
image3.push(name)
docker.withRegistry(
'https://us-central1-docker.pkg.dev/',
'google_archive_registry_sa'
) {
registryTags.each { name ->
image2.push(name+arch)
}
}
}
docker.withRegistry(
'https://us-central1-docker.pkg.dev/',
'google_archive_registry_sa'
) {
registryTags.each { name ->
image2.push(name)
}
}
digest = sh(
script: "docker inspect --format='{{index .RepoDigests 0}}' ${dockerRepo}:${dockerTag}",
returnStdout: true
).trim()
digest = sh(
script: "docker inspect --format='{{index .RepoDigests 0}}' ${dockerRepo}:${dockerTag}",
returnStdout: true
).trim()

}
println(digest)
}
dockerdigest.add(digest)
} // push
} // push

} // run

Expand All @@ -163,26 +163,26 @@ notify.wrap {
run()
}
} finally {
stage('archive') {
def resultsFile = 'results.json'

util.dumpJson(resultsFile, [
base_image: baseImage ?: null,
image: "${dockerRepo}:${dockerTag}",
docker_registry: [
repo: dockerRepo,
tag: dockerTag
],
])

archiveArtifacts([
artifacts: resultsFile,
fingerprint: true
])
} // stage
} // try
} // util.nodeWrap
}
stage('archive') {
def resultsFile = 'results.json'

util.dumpJson(resultsFile, [
base_image: baseImage ?: null,
image: "${dockerRepo}:${dockerTag}",
docker_registry: [
repo: dockerRepo,
tag: dockerTag
],
])

archiveArtifacts([
artifacts: resultsFile,
fingerprint: true
])
} // stage
} // try
} // util.nodeWrap
}
}
parallel matrix

Expand Down Expand Up @@ -210,11 +210,6 @@ notify.wrap {
returnStdout: true)
}

sh(script: """ \
docker buildx imagetools create -t $dockerRepo:$dockerTag \
$digest
""",
returnStdout: true)
}
docker.withRegistry(
'https://us-central1-docker.pkg.dev/',
Expand Down
33 changes: 15 additions & 18 deletions pipelines/release/nightly_release.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ notify.wrap {
def products = scipipe.canonical.products
def tarballProducts = scipipe.tarball.products
def retries = 3
def extraDockerTags = null //'7-stack-lsst_distrib-d_latest d_latest'
def extraDockerTags = '7-stack-lsst_distrib-d_latest d_latest'

def gitTag = null
def eupsTag = null
Expand All @@ -35,7 +35,7 @@ notify.wrap {

def run = {
stage('format nightly tag') {
gitTag = "exp.d.${year}.${month}.${day}"
gitTag = "d.${year}.${month}.${day}"
eupsTag = util.sanitizeEupsTag(gitTag)
echo "generated [git] tag: ${gitTag}"
echo "generated [eups] tag: ${eupsTag}"
Expand All @@ -55,9 +55,7 @@ notify.wrap {
stage('eups publish') {
def pub = [:]

[eupsTag,
//'d_latest'
].each { tagName ->
[eupsTag,'d_latest'].each { tagName ->
pub[tagName] = {
retry(retries) {
util.runPublish(
Expand All @@ -66,7 +64,6 @@ notify.wrap {
MANIFEST_ID: manifestId,
EUPS_TAG: tagName,
PRODUCTS: products,
BUILD_DOCS: true,
],
)
} // retry
Expand Down Expand Up @@ -145,18 +142,18 @@ notify.wrap {

def triggerMe = [:]

//triggerMe['build Science Platform Notebook Aspect Lab image'] = {
// retry(retries) {
// // based on lsstsqre/stack image
// build(
// job: 'sqre/infra/build-sciplatlab',
// parameters: [
// string(name: 'TAG', value: eupsTag),
// ],
// wait: false,
// )
// } // retry
//}
triggerMe['build Science Platform Notebook Aspect Lab image'] = {
retry(retries) {
// based on lsstsqre/stack image
build(
job: 'sqre/infra/build-sciplatlab',
parameters: [
string(name: 'TAG', value: eupsTag),
],
wait: false,
)
} // retry
}
stackResults = ['image':'lsstsqre/centos:7-stack-lsst_distrib-exp_d_2025_01_22']

triggerMe['verify_drp_metrics x86'] = {
Expand Down
7 changes: 5 additions & 2 deletions pipelines/release/run_publish.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,10 @@ notify.wrap {
if (params.RUBINENV_VER) {
rubinEnvVer = params.RUBINENV_VER
}

def slug = util.lsstswConfigSlug(lsstswConfig)
def run = {

def run = {
ws(canonical.workspace) {
def cwd = pwd()
def pkgroot = "${cwd}/distrib"
Expand Down Expand Up @@ -96,6 +98,7 @@ notify.wrap {
# this can be retrived using the -b option.
# (note: bin/setup.sh is now deprecated)
source ./lsstsw/bin/envconfig -n "lsst-scipipe-$LSST_SPLENV_REF"
publish "${ARGS[@]}"
'''
}
Expand Down Expand Up @@ -139,7 +142,7 @@ notify.wrap {
} // stage('push packages')
} // ws
} // run
util.nodeWrap(lsstswConfig.label) {
util.nodeWrap(lsstswConfig.label) {
timeout(time: timelimit, unit: 'HOURS') {
run()
}
Expand Down
Loading

0 comments on commit b90fe10

Please sign in to comment.