From e0c1b113ddd3e038295d7240e4619a8f6d96fe1f Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 24 Oct 2024 16:23:40 -0400 Subject: [PATCH] Add retry for windows on min snapshot and distribution (#5132) Signed-off-by: Peter Zhu --- .../distribution-build.jenkinsfile | 38 ++++++----- .../opensearch/distribution-build.jenkinsfile | 44 +++++++------ .../publish-min-snapshots.jenkinsfile | 32 ++++----- .../publish-min-snapshots.jenkinsfile.txt | 65 ++++++++++--------- 4 files changed, 93 insertions(+), 86 deletions(-) diff --git a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile index d34eb2c958..7bf3e6774e 100644 --- a/jenkins/opensearch-dashboards/distribution-build.jenkinsfile +++ b/jenkins/opensearch-dashboards/distribution-build.jenkinsfile @@ -819,26 +819,28 @@ pipeline { } steps { script { - def buildManifestObj = buildAssembleUpload( - componentName: "${COMPONENT_NAME}", - inputManifest: "manifests/${INPUT_MANIFEST}", - platform: 'windows', - architecture: 'x64', - distribution: 'zip', - continueOnError: params.CONTINUE_ON_ERROR, - incremental: params.INCREMENTAL, - previousBuildId: params.PREVIOUS_BUILD_ID - ) - String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) - String buildManifestUrlOpenSearch = [buildManifestObj.getArtifactRootUrl(JOB_NAME_OPENSEARCH, 'latest'), 'builds', 'opensearch', 'manifest.yml'].join('/') - String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) - env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl - env.INDEX_FILE_PATH_X64_ZIP = buildManifestObj.getIndexFileRoot("${JOB_NAME}") + retry(3) { + def buildManifestObj = buildAssembleUpload( + componentName: "${COMPONENT_NAME}", + inputManifest: "manifests/${INPUT_MANIFEST}", + platform: 'windows', + architecture: 'x64', + distribution: 'zip', + continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID + ) + String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) + String buildManifestUrlOpenSearch = [buildManifestObj.getArtifactRootUrl(JOB_NAME_OPENSEARCH, 'latest'), 'builds', 'opensearch', 'manifest.yml'].join('/') + String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) + env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl + env.INDEX_FILE_PATH_X64_ZIP = buildManifestObj.getIndexFileRoot("${JOB_NAME}") - echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}" - echo "artifactUrl (windows, x64, zip): ${artifactUrl}" + echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}" + echo "artifactUrl (windows, x64, zip): ${artifactUrl}" - triggerIntegrationTests(buildManifestUrl, buildManifestUrlOpenSearch, 'windows', 'zip') + triggerIntegrationTests(buildManifestUrl, buildManifestUrlOpenSearch, 'windows', 'zip') + } } } post { diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index 720505ee9c..291ea6a82f 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -780,27 +780,29 @@ pipeline { } steps { script { - echo("Switching to Java ${env.javaVersionNumber} on Windows Docker Container") - sh("scoop reset `scoop list jdk | cut -d ' ' -f1 | grep ${env.javaVersionNumber} | head -1`") - def buildManifestObj = buildAssembleUpload( - componentName: "${COMPONENT_NAME}", - inputManifest: "manifests/${INPUT_MANIFEST}", - platform: 'windows', - architecture: 'x64', - distribution: 'zip', - continueOnError: params.CONTINUE_ON_ERROR, - incremental: params.INCREMENTAL, - previousBuildId: params.PREVIOUS_BUILD_ID - ) - String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) - String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) - env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl - env.INDEX_FILE_PATH_X64_ZIP = buildManifestObj.getIndexFileRoot("${JOB_NAME}") - - echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}" - echo "artifactUrl (windows, x64, zip): ${artifactUrl}" - - triggerIntegrationTests(buildManifestUrl, 'windows', 'zip') + retry(3) { + echo("Switching to Java ${env.javaVersionNumber} on Windows Docker Container") + sh("scoop reset `scoop list jdk | cut -d ' ' -f1 | grep ${env.javaVersionNumber} | head -1`") + def buildManifestObj = buildAssembleUpload( + componentName: "${COMPONENT_NAME}", + inputManifest: "manifests/${INPUT_MANIFEST}", + platform: 'windows', + architecture: 'x64', + distribution: 'zip', + continueOnError: params.CONTINUE_ON_ERROR, + incremental: params.INCREMENTAL, + previousBuildId: params.PREVIOUS_BUILD_ID + ) + String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) + String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) + env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl + env.INDEX_FILE_PATH_X64_ZIP = buildManifestObj.getIndexFileRoot("${JOB_NAME}") + + echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}" + echo "artifactUrl (windows, x64, zip): ${artifactUrl}" + + triggerIntegrationTests(buildManifestUrl, 'windows', 'zip') + } } } post { diff --git a/jenkins/opensearch/publish-min-snapshots.jenkinsfile b/jenkins/opensearch/publish-min-snapshots.jenkinsfile index 07339f927f..fafa9cd609 100644 --- a/jenkins/opensearch/publish-min-snapshots.jenkinsfile +++ b/jenkins/opensearch/publish-min-snapshots.jenkinsfile @@ -193,21 +193,23 @@ pipeline { } steps { script { - echo("Switching to Java ${env.javaVersionNumber} on Windows Docker Container") - sh("scoop reset `scoop list jdk | cut -d ' ' -f1 | grep ${env.javaVersionNumber} | head -1`") - buildManifest( - componentName: "OpenSearch", - inputManifest: "manifests/${INPUT_MANIFEST}", - platform: 'windows', - architecture: 'x64', - distribution: 'zip', - snapshot: true - ) - echo("Uploading windows x64 zip min snapshots to S3") - uploadMinSnapshotsToS3( - fileActions: [createSha512Checksums()], - distribution: 'zip' - ) + retry(3) { + echo("Switching to Java ${env.javaVersionNumber} on Windows Docker Container") + sh("scoop reset `scoop list jdk | cut -d ' ' -f1 | grep ${env.javaVersionNumber} | head -1`") + buildManifest( + componentName: "OpenSearch", + inputManifest: "manifests/${INPUT_MANIFEST}", + platform: 'windows', + architecture: 'x64', + distribution: 'zip', + snapshot: true + ) + echo("Uploading windows x64 zip min snapshots to S3") + uploadMinSnapshotsToS3( + fileActions: [createSha512Checksums()], + distribution: 'zip' + ) + } } } post { diff --git a/tests/jenkins/jenkinsjob-regression-files/opensearch/publish-min-snapshots.jenkinsfile.txt b/tests/jenkins/jenkinsjob-regression-files/opensearch/publish-min-snapshots.jenkinsfile.txt index 6c6e2a6ccd..ff2887fcbe 100644 --- a/tests/jenkins/jenkinsjob-regression-files/opensearch/publish-min-snapshots.jenkinsfile.txt +++ b/tests/jenkins/jenkinsjob-regression-files/opensearch/publish-min-snapshots.jenkinsfile.txt @@ -183,42 +183,43 @@ publish-min-snapshots.stage(windows-x64-zip, groovy.lang.Closure) publish-min-snapshots.echo(Executing on agent [docker:[alwaysPull:true, args:, containerPerStageRoot:false, label:Jenkins-Agent-Windows2019-X64-C54xlarge-Docker-Host, image:opensearchstaging/ci-runner:ci-runner-windows2019-opensearch-build-v1, reuseNode:false, registryUrl:https://public.ecr.aws/, stages:[:]]]) publish-min-snapshots.script(groovy.lang.Closure) - publish-min-snapshots.echo(Switching to Java 17 on Windows Docker Container) - publish-min-snapshots.sh(scoop reset `scoop list jdk | cut -d ' ' -f1 | grep 17 | head -1`) - publish-min-snapshots.buildManifest({componentName=OpenSearch, inputManifest=manifests/3.0.0/opensearch-3.0.0.yml, platform=windows, architecture=x64, distribution=zip, snapshot=true}) - buildManifest.legacySCM(groovy.lang.Closure) - buildManifest.library({identifier=jenkins@6.6.1, retriever=null}) - buildManifest.readYaml({file=manifests/3.0.0/opensearch-3.0.0.yml}) - InputManifest.asBoolean() - buildManifest.sh(./build.sh manifests/3.0.0/opensearch-3.0.0.yml -d zip --component OpenSearch -p windows -a x64 --snapshot) - publish-min-snapshots.echo(Uploading windows x64 zip min snapshots to S3) - publish-min-snapshots.createSha512Checksums() - publish-min-snapshots.uploadMinSnapshotsToS3({fileActions=[groovy.lang.Closure], distribution=zip}) - uploadMinSnapshotsToS3.legacySCM(groovy.lang.Closure) - uploadMinSnapshotsToS3.library({identifier=jenkins@6.6.1, retriever=null}) - uploadMinSnapshotsToS3.readYaml({file=manifests/3.0.0/opensearch-3.0.0.yml}) - InputManifest.asBoolean() - uploadMinSnapshotsToS3.echo(Retreving build manifest from: /tmp/workspace/zip/builds/opensearch/manifest.yml) - uploadMinSnapshotsToS3.readYaml({file=/tmp/workspace/zip/builds/opensearch/manifest.yml}) - BuildManifest.asBoolean() - uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts) - createSha512Checksums.sh({script=find /tmp/workspace/zip/builds/opensearch/dist -type f, returnStdout=true}) - createSha512Checksums.echo(Not generating sha for bbb in /tmp/workspace/zip/builds/opensearch/dist, doesn't match allowed types [.tar.gz, .zip, .rpm, .deb]) - createSha512Checksums.echo(Not generating sha for ccc in /tmp/workspace/zip/builds/opensearch/dist, doesn't match allowed types [.tar.gz, .zip, .rpm, .deb]) - uploadMinSnapshotsToS3.echo(Start copying files: version-3.0.0-SNAPSHOT architecture-x64 platform-windows buildid-8469 distribution-zip extension-zip) - uploadMinSnapshotsToS3.sh( + publish-min-snapshots.retry(3, groovy.lang.Closure) + publish-min-snapshots.echo(Switching to Java 17 on Windows Docker Container) + publish-min-snapshots.sh(scoop reset `scoop list jdk | cut -d ' ' -f1 | grep 17 | head -1`) + publish-min-snapshots.buildManifest({componentName=OpenSearch, inputManifest=manifests/3.0.0/opensearch-3.0.0.yml, platform=windows, architecture=x64, distribution=zip, snapshot=true}) + buildManifest.legacySCM(groovy.lang.Closure) + buildManifest.library({identifier=jenkins@6.6.1, retriever=null}) + buildManifest.readYaml({file=manifests/3.0.0/opensearch-3.0.0.yml}) + InputManifest.asBoolean() + buildManifest.sh(./build.sh manifests/3.0.0/opensearch-3.0.0.yml -d zip --component OpenSearch -p windows -a x64 --snapshot) + publish-min-snapshots.echo(Uploading windows x64 zip min snapshots to S3) + publish-min-snapshots.createSha512Checksums() + publish-min-snapshots.uploadMinSnapshotsToS3({fileActions=[groovy.lang.Closure], distribution=zip}) + uploadMinSnapshotsToS3.legacySCM(groovy.lang.Closure) + uploadMinSnapshotsToS3.library({identifier=jenkins@6.6.1, retriever=null}) + uploadMinSnapshotsToS3.readYaml({file=manifests/3.0.0/opensearch-3.0.0.yml}) + InputManifest.asBoolean() + uploadMinSnapshotsToS3.echo(Retreving build manifest from: /tmp/workspace/zip/builds/opensearch/manifest.yml) + uploadMinSnapshotsToS3.readYaml({file=/tmp/workspace/zip/builds/opensearch/manifest.yml}) + BuildManifest.asBoolean() + uploadMinSnapshotsToS3.echo(Create .sha512 for Min Snapshots Artifacts) + createSha512Checksums.sh({script=find /tmp/workspace/zip/builds/opensearch/dist -type f, returnStdout=true}) + createSha512Checksums.echo(Not generating sha for bbb in /tmp/workspace/zip/builds/opensearch/dist, doesn't match allowed types [.tar.gz, .zip, .rpm, .deb]) + createSha512Checksums.echo(Not generating sha for ccc in /tmp/workspace/zip/builds/opensearch/dist, doesn't match allowed types [.tar.gz, .zip, .rpm, .deb]) + uploadMinSnapshotsToS3.echo(Start copying files: version-3.0.0-SNAPSHOT architecture-x64 platform-windows buildid-8469 distribution-zip extension-zip) + uploadMinSnapshotsToS3.sh( cp -v /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64.zip /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip cp -v /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64.zip.sha512 /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512 cp -v /tmp/workspace/zip/builds/opensearch/dist/../manifest.yml /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.build-manifest.yml sed -i "s/.zip/-latest.zip/g" /tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512 ) - uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) - uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) - uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) - uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) - uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) - uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip}) - uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512}) - uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.build-manifest.yml}) + uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME}) + uploadMinSnapshotsToS3.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT}) + uploadMinSnapshotsToS3.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME}) + uploadMinSnapshotsToS3.withCredentials([ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure) + uploadMinSnapshotsToS3.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure) + uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip}) + uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.sha512}) + uploadMinSnapshotsToS3.s3Upload({file=/tmp/workspace/zip/builds/opensearch/dist/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.build-manifest.yml, bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=snapshots/core/opensearch/3.0.0-SNAPSHOT/opensearch-min-3.0.0-SNAPSHOT-windows-x64-latest.zip.build-manifest.yml}) publish-min-snapshots.postCleanup() postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})