Skip to content

Commit

Permalink
NO-ISSUE: Jenkins jobs updates and fixes for Apache 10 release (#1804)
Browse files Browse the repository at this point in the history 
* Adjust release pipelines for the Apache 10 release (#1789)

* kie-issues#1397 - Adjust git tag creation for Apache 10 release  (#1790)

* Adjust git tag creation for Apache 10 release

* Add image suffix to the images artifact

* Fix release utils (#1792)

* Kogito Images release deploy job fixes (#1799)

* Push missing checksum file to the SVN repository (#1800)
  • Loading branch information
@rodrigonull
rodrigonull authored Oct 7, 2024
1 parent c7083c5 commit b813385
Show file tree
Hide file tree
Showing 5 changed files with 66 additions and 99 deletions.
44 changes: 29 additions & 15 deletions .ci/jenkins/Jenkinsfile.build-image
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,19 +87,23 @@ pipeline {

if (getMavenArtifactRepository()) {
echo "Setup Repository url to '${getMavenArtifactRepository()}'"
runPythonCommand("python scripts/update-repository.py --repo-url ${getMavenArtifactRepository()}")
runPythonCommand("python scripts/update-repository.py --repo-urls ${getMavenArtifactRepository()}")
}
}
}
}

stage('Update kogito version') {
stage('Update project version') {
when {
expression { params.UPDATE_KOGITO_VERSION != '' }
}
steps {
script {
runPythonCommand("python scripts/manage-kogito-version.py --bump-to ${params.UPDATE_KOGITO_VERSION} --confirm")
versionCmd = "python scripts/manage-kogito-version.py --bump-to ${params.UPDATE_KOGITO_VERSION} --confirm"
if (isRelease()) {
versionCmd += " --examples-ref ${getSourceBranch()}"
}
sh versionCmd
}
}
}
Expand Down Expand Up @@ -128,7 +132,7 @@ pipeline {

int freePort = cloud.findFreePort()
env.localRegistryUrl = cloud.startLocalRegistry(freePort)
cloud.prepareForDockerMultiplatformBuild([env.localRegistryUrl],[cloud.getDockerIOMirrorRegistryConfig()], false)
cloud.prepareForDockerMultiplatformBuild([env.localRegistryUrl], [cloud.getDockerIOMirrorRegistryConfig()], false)

// Generate the Dockerfile
runPythonCommand("make build-image ${getMakeBuildImageArgs()} image_name=${getBuildImageName()} ignore_test=true ignore_tag=true build_options='--dry-run'")
Expand Down Expand Up @@ -184,7 +188,7 @@ pipeline {
script {
updateGithubCommitStatus('PENDING', 'Tests in progress')

String testImageTag = "docker.io/apache/incubator-kie-${getBuildImageName()}:${getImageVersion()}"
String testImageTag = "${getDeployImageRegistry()}/${getDeployImageNamespace()}/incubator-kie-${getBuildImageName()}:${getImageVersion()}"
String builtImageTag = getBuiltImageTag()
// Pull and tag to test image
sh """
Expand All @@ -208,15 +212,17 @@ pipeline {
}
steps {
script {
def resultingFileName = "incubator-kie-${getImageArtifactReleaseVersion()}-${getBuildImageName()}.tar.gz"
def signatureFileName = "${resultingFileName}.asc"
String resultingFileName = "incubator-kie-${getImageArtifactReleaseVersion()}-${getBuildImageName()}-image.tar.gz"
String signatureFileName = "${resultingFileName}.asc"
String checksumFileName = "${resultingFileName}.sha512"
sh """
docker pull ${getBuiltImageTag()}
docker save ${getBuiltImageTag()} | gzip > ${resultingFileName}
docker tag ${getBuiltImageTag()} ${getBuiltDeployImageTag()}
docker save ${getBuiltDeployImageTag()} | gzip > ${resultingFileName}
"""
release.gpgImportKeyFromStringWithoutPassword(getReleaseGpgSignKeyCredsId())
release.gpgSignFileDetachedSignatureWithoutPassword(resultingFileName, signatureFileName)
release.svnUploadFileToRepository(getReleaseSvnRepository(), getReleaseSvnCredsId(), getImageArtifactReleaseVersion(), resultingFileName, signatureFileName)
releaseUtils.gpgImportKeyFromStringWithoutPassword(getReleaseGpgSignKeyCredsId())
releaseUtils.gpgSignFileDetachedSignatureWithoutPassword(resultingFileName, signatureFileName)
releaseUtils.svnUploadFileToRepository(getReleaseSvnRepository(), getReleaseSvnCredsId(), getImageArtifactReleaseVersion(), resultingFileName, signatureFileName, checksumFileName)
}
}
post {
Expand Down Expand Up @@ -269,9 +275,13 @@ String getImageVersion() {
return runPythonCommand('make display-image-version', true).trim()
}

String getBuiltDeployImageTag(String imageTag = '') {
return "${getDeployImageRegistry()}/${getDeployImageNamespace()}/${getFinalImageName()}:${imageTag ?: getDeployImageTag()}"
}

String getBuiltImageTag(String imageTag = '') {
if (shouldDeployImage()) {
return "${getDeployImageRegistry()}/${getDeployImageNamespace()}/${getFinalImageName()}:${imageTag ?: getDeployImageTag()}"
return getBuiltDeployImageTag(imageTag)
}
return "${env.localRegistryUrl}/${getBuildImageName()}:${githubscm.getCommitHash()}"
}
Expand Down Expand Up @@ -316,7 +326,7 @@ String getDeployImageTag() {
}

String getFinalImageName() {
return "incubator-kie-" + getBuildImageName() + (getDeployImageNameSuffix() ? "-${getDeployImageNameSuffix()}" : '')
return 'incubator-kie-' + getBuildImageName() + (getDeployImageNameSuffix() ? "-${getDeployImageNameSuffix()}" : '')
}

boolean isDeployLatestTag() {
Expand Down Expand Up @@ -440,7 +450,7 @@ String getTriggeringProjectName() {
if (currentBuild.upstreamBuilds) {
return currentBuild.upstreamBuilds[0].projectName
}
return ""
return ''
}
String getReleaseGpgSignKeyCredsId() {
return env.RELEASE_GPG_SIGN_KEY_CREDS_ID
Expand All @@ -459,5 +469,9 @@ String getReleaseSvnCredsId() {
}

String getImageArtifactReleaseVersion() {
return env.IMAGE_ARTIFACT_RELEASE_VERSION
return params.IMAGE_ARTIFACT_RELEASE_VERSION
}

boolean isRelease() {
return env.RELEASE ? env.RELEASE.toBoolean() : false
}
96 changes: 16 additions & 80 deletions .ci/jenkins/Jenkinsfile.deploy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,13 @@

deployProperties = [:]

changesDone = false

BUILT_IMAGES = Collections.synchronizedList([])
BUILD_FAILED_IMAGES = Collections.synchronizedList([])
TEST_FAILED_IMAGES = Collections.synchronizedList([])

pipeline {
agent {
docker {
docker {
image env.AGENT_DOCKER_BUILDER_IMAGE
args env.AGENT_DOCKER_BUILDER_ARGS
label util.avoidFaultyNodes()
Expand Down Expand Up @@ -73,22 +71,6 @@ pipeline {
}
}

stage('Prepare for PR') {
when {
expression { return isRelease() }
}
steps {
script {
dir(getRepoName()) {
if (githubscm.isBranchExist('origin', getPRBranch())) {
githubscm.removeRemoteBranch('origin', getPRBranch(), getGitAuthorPushCredsId())
}
githubscm.createBranch(getPRBranch())
}
}
}
}

stage('Setup Quarkus platform version') {
when {
expression { isRelease() }
Expand Down Expand Up @@ -117,14 +99,12 @@ pipeline {
dir(getRepoName()) {
versionCmd = "python scripts/manage-kogito-version.py --bump-to ${getProjectVersion()} --confirm"
if (getBuildBranch() != 'main') {
versionCmd += " --examples-ref ${!isRelease() ? 'nightly-' : ''}${getBuildBranch()}"
versionCmd += " --examples-ref ${getBuildBranch()}"
}
if (getKogitoArtifactsVersion()) {
versionCmd += " --artifacts-version ${getKogitoArtifactsVersion()}"
}
sh versionCmd

commitAndPushChanges("Update project version to ${getProjectVersion()}")
}
}
}
Expand Down Expand Up @@ -173,62 +153,16 @@ pipeline {
}
}

stage('Create PR') {
stage('Commit and Tag changes') {
when {
expression {
return changesDone && isRelease()
}
expression { return isRelease() }
}
steps {
script {
dir(getRepoName()) {
def commitMsg = "[${getBuildBranch()}] Update Maven artifacts"
def prBody = "Generated by build ${BUILD_TAG}: ${BUILD_URL}."
if (isRelease()) {
commitMsg = "[${getBuildBranch()}] Update project version to ${getProjectVersion()}"
prBody += '\nPlease do not merge, it should be merged automatically.'
} else if (currentBuild.currentResult != 'SUCCESS') {
commitMsg += " (${currentBuild.currentResult})"
prBody += '\n\nSuccessful images:\n'
getBuiltImages().each {
prBody += "- ${it}\n"
}
if (getBuildFailedImages()) {
prBody += '\nBuild failures on those images:\n'
getBuildFailedImages().each {
prBody += "- ${it}\n"
}
} else {
prBody += '\nImages were all successfully built but some other problem occured in the pipeline execution...\n'
}
if (getTestFailedImages()) {
prBody += '\nTest failures on those images:\n'
getTestFailedImages().each {
prBody += "- ${it}\n"
}
} else {
prBody += '\nImages were all successfully built but some other problem occured in the pipeline execution...\n'
}
prBody += '\nSee build url above for more information'
if (!getBuildFailedImages()) {
dir(getRepoName()) {
commitAndTagChanges("[${getBuildBranch()}] Update version to ${getProjectVersion()}")
}
String prLink = githubscm.createPR(commitMsg, prBody, getBuildBranch(), getGitAuthorPushCredsId())
deployProperties["${getRepoName()}.pr.link"] = prLink

// Create a new tag
githubscm.setUserConfigFromCreds(getGitAuthorPushCredsId())
githubscm.tagRepository(getGitTagName())
githubscm.pushRemoteTag('origin', getGitTagName(), getGitAuthorPushCredsId())
}
}
}
post {
always {
script {
setDeployPropertyIfNeeded("${getRepoName()}.pr.source.uri", "https://github.com/${getGitAuthor()}/${getRepoName()}")
setDeployPropertyIfNeeded("${getRepoName()}.pr.source.ref", getPRBranch())
setDeployPropertyIfNeeded("${getRepoName()}.pr.target.uri", "https://github.com/${getGitAuthor()}/${getRepoName()}")
setDeployPropertyIfNeeded("${getRepoName()}.pr.target.ref", getBuildBranch())
setDeployPropertyIfNeeded("${getRepoName()}.tag.name", getGitTagName())
}
}
}
Expand Down Expand Up @@ -284,11 +218,11 @@ void checkoutRepo() {
checkout(githubscm.resolveRepository(getRepoName(), getGitAuthor(), getBuildBranch(), false, getGitAuthorCredsId()))
}

void commitAndPushChanges(String commitMsg) {
void commitAndTagChanges(String commitMsg) {
githubscm.setUserConfigFromCreds(getGitAuthorPushCredsId())
githubscm.commitChanges(commitMsg)
githubscm.pushObject('origin', getPRBranch(), getGitAuthorPushCredsId())
changesDone = true
githubscm.tagRepository(getGitTagName())
githubscm.pushRemoteTag('origin', getGitTagName(), getGitAuthorPushCredsId())
}

void createBuildAndTestStageClosure(String image) {
Expand All @@ -297,16 +231,18 @@ void createBuildAndTestStageClosure(String image) {
List buildParams = []
buildParams.add(string(name: 'DISPLAY_NAME', value: "${params.DISPLAY_NAME} - ${image}"))
buildParams.add(string(name: 'BUILD_IMAGE_NAME', value: image))
buildParams.add(string(name: 'SOURCE_AUTHOR', value: isRelease() ? getGitAuthor() : getGitAuthor()))
buildParams.add(string(name: 'SOURCE_BRANCH', value: isRelease() ? getPRBranch() : getBuildBranch()))
buildParams.add(string(name: 'SOURCE_AUTHOR', value: getGitAuthor()))
buildParams.add(string(name: 'SOURCE_BRANCH', value: getBuildBranch()))

buildParams.add(string(name: 'MAVEN_ARTIFACTS_REPOSITORY', value: env.MAVEN_ARTIFACT_REPOSITORY ?: (isRelease() ? env.DEFAULT_STAGING_REPOSITORY : '')))
buildParams.add(string(name: 'MAVEN_ARTIFACTS_REPOSITORY', value: env.MAVEN_ARTIFACT_REPOSITORY))
buildParams.add(string(name: 'BUILD_KOGITO_APPS_URI', value: params.APPS_URI))
buildParams.add(string(name: 'BUILD_KOGITO_APPS_REF', value: params.APPS_REF))
buildParams.add(booleanParam(name: 'SKIP_TESTS', value: params.SKIP_TESTS))
buildParams.add(string(name: 'TESTS_KOGITO_EXAMPLES_URI', value: params.EXAMPLES_URI))
buildParams.add(string(name: 'TESTS_KOGITO_EXAMPLES_REF', value: params.EXAMPLES_REF))

buildParams.add(string(name: 'UPDATE_KOGITO_VERSION', value: isRelease() ? getProjectVersion() : ''))

buildParams.add(booleanParam(name: 'DEPLOY_IMAGE', value: isDeployImage()))
buildParams.add(booleanParam(name: 'DEPLOY_IMAGE_USE_OPENSHIFT_REGISTRY', value: isDeployImageInOpenshiftRegistry()))
buildParams.add(string(name: 'DEPLOY_IMAGE_REGISTRY_USER_CREDENTIALS_ID', value: getDeployImageRegistryUserCredentialsId()))
Expand Down Expand Up @@ -364,7 +300,7 @@ List getTestFailedImages() {
////////////////////////////////////////////////////////////////////////

boolean isDeployImage() {
return !Boolean.valueOf(env.DISABLE_IMAGES_DEPLOY)
return !env.DISABLE_IMAGES_DEPLOY
}

boolean isDeployImageInOpenshiftRegistry() {
Expand Down
8 changes: 4 additions & 4 deletions .ci/jenkins/dsl/jobs.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,8 +151,8 @@ void setupDeployJob(JobType jobType) {
stringParam('KOGITO_ARTIFACTS_VERSION', '', 'Optional. If artifacts\' version is different from PROJECT_VERSION.')
if (jobType == JobType.RELEASE) {
stringParam('QUARKUS_PLATFORM_VERSION', '', 'Allow to override the Quarkus Platform version')
stringParam('GIT_TAG_NAME', '', 'Git tag to be created')
}
stringParam('GIT_TAG_NAME', '', 'Optional if not RELEASE. Tag to be created in the repository')

stringParam('KOGITO_PR_BRANCH', '', 'PR branch name')
booleanParam('SEND_NOTIFICATION', false, 'In case you want the pipeline to send a notification on CI channel for this run.')
Expand All @@ -176,8 +176,8 @@ void setupBuildImageJob(JobType jobType) {

RELEASE_GPG_SIGN_KEY_CREDS_ID: Utils.getReleaseGpgSignKeyCredentialsId(this),
RELEASE_GPG_SIGN_PASSPHRASE_CREDS_ID: Utils.getReleaseGpgSignPassphraseCredentialsId(this),
RELEASE_SVN_REPOSITORY: Utils.getReleaseSvnCredentialsId(this),
RELEASE_SVN_CREDS_ID: Utils.getReleaseSvnStagingRepository(this)
RELEASE_SVN_REPOSITORY: Utils.getReleaseSvnStagingRepository(this),
RELEASE_SVN_CREDS_ID: Utils.getReleaseSvnCredentialsId(this)
])
KogitoJobTemplate.createPipelineJob(this, jobParams)?.with {
logRotator {
Expand Down Expand Up @@ -216,7 +216,7 @@ void setupBuildImageJob(JobType jobType) {
stringParam('DEPLOY_IMAGE_TAG', '', 'Image tag to use to deploy images')
booleanParam('DEPLOY_WITH_LATEST_TAG', false, 'Set to true if you want the deployed images to also be with the `latest` tag')
booleanParam('EXPORT_AND_GPG_SIGN_IMAGE', jobType == JobType.RELEASE, 'Set to true if should images be exported and signed.')
stringParam('IMAGE_ARTIFACT_RELEASE_VERSION', '', 'Set the release version to be attached to the images artifacts names')
stringParam('IMAGE_ARTIFACT_RELEASE_VERSION', '', 'Optional if not RELEASE. Set the release version to be attached to the images artifacts names')
}
}
}
Expand Down
13 changes: 13 additions & 0 deletions scripts/common.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -430,6 +430,19 @@ def update_maven_repo_in_setup_maven(repo_url, replace_default_repository):
replacement = 'export MAVEN_REPO_URL="{}"'.format(repo_url)
update_in_file(SETUP_MAVEN_SCRIPT, pattern, replacement)

def update_maven_repos_in_setup_maven(repos_url):
"""
Update maven repositories into setup-maven.sh script
:param repos_url: Maven repositories urls
"""
repo_list = repos_url.split(',')
print("Set maven repos {} in setup-maven script".format(repo_list))
pattern = re.compile(r'(# export MAVEN_REPO_URL=.*)')
replacement = f"export MAVEN_REPOS={','.join(['REPO_' + str(i) for i, _ in enumerate(repo_list)])}\n"
for i, value in enumerate(repo_list):
replacement += f"export REPO_{i}_MAVEN_REPO_URL={value}\n"
update_in_file(SETUP_MAVEN_SCRIPT, pattern, replacement)

def update_env_value(env_name, env_value):
"""
Update environment value into the given yaml module/image file
Expand Down
4 changes: 4 additions & 0 deletions scripts/update-repository.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Update Maven information in repo from the given artifact url and '
'version.')
parser.add_argument('--repo-urls', dest='repo_urls', help='Defines the urls of the repositories to setup maven, comma (,) separated')
parser.add_argument('--repo-url', dest='repo_url', help='Defines the url of the repository to setup into the tests')
parser.add_argument('--replace-default-repo', dest='replace_default_repo', default=False, action='store_true',
help='Enable if repo-url should replace the default repository')
Expand All @@ -53,6 +54,9 @@
parser.add_argument('--tests-only', dest='tests_only', default=False, action='store_true', help='Update product modules/images')
args = parser.parse_args()

if args.repo_urls:
common.update_maven_repos_in_setup_maven(args.repo_urls)

if args.repo_url:
common.update_maven_repo_in_build_config(args.repo_url, args.replace_default_repo)
common.update_maven_repo_in_setup_maven(args.repo_url, args.replace_default_repo)
Expand Down

0 comments on commit b813385

Please sign in to comment.