From ed5874bc27a61ab22bcbc40111795cdde5245790 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Tue, 29 Oct 2024 10:55:15 +0100 Subject: [PATCH] Use jvm catalog for reproducible builds and expose new pipeline to check JDK availability (#16602) Updates the existing `createElasticCatalogDownloadUrl` method to use the precise version retrieved `versions.yml` to download the JDK instead of using the latest of major version. This makes the build reproducible again. Defines a new Gradle `checkNewJdkVersion` task to check if there is a new JDK version available from JVM catalog matching the same major of the current branch. Creates a new Buildkite pipeline to execute a `bash` script to run the Gradle task; plus it also update the `catalog-info.yaml` with the new pipeline and a trigger to execute every week. --- .../jdk_availability_check_pipeline.yml | 7 ++ build.gradle | 69 +++++++++++++------ catalog-info.yaml | 49 +++++++++++++ ci/check_jdk_version_availability.sh | 7 ++ 4 files changed, 112 insertions(+), 20 deletions(-) create mode 100644 .buildkite/jdk_availability_check_pipeline.yml create mode 100755 ci/check_jdk_version_availability.sh diff --git a/.buildkite/jdk_availability_check_pipeline.yml b/.buildkite/jdk_availability_check_pipeline.yml new file mode 100644 index 00000000000..f4baaec0a0b --- /dev/null +++ b/.buildkite/jdk_availability_check_pipeline.yml @@ -0,0 +1,7 @@ +steps: + - label: "JDK Availability check" + command: | + set -euo pipefail + + source .buildkite/scripts/common/vm-agent.sh + ci/check_jdk_version_availability.sh \ No newline at end of file diff --git a/build.gradle b/build.gradle index 979513ca257..d80d26b1798 100644 --- a/build.gradle +++ b/build.gradle @@ -740,46 +740,66 @@ class JDKDetails { return createElasticCatalogDownloadUrl() } + // throws an error iff local version in versions.yml doesn't match the latest from JVM catalog. + void checkLocalVersionMatchingLatest() { + // retrieve the metadata from remote + def url = "https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_${major}_${osName}" + def catalogMetadataUrl = URI.create(url).toURL() + def catalogConnection = catalogMetadataUrl.openConnection() + catalogConnection.requestMethod = 'GET' + assert catalogConnection.responseCode == 200 + + def metadataRetrieved = catalogConnection.content.text + def catalogMetadata = new JsonSlurper().parseText(metadataRetrieved) + + if (catalogMetadata.version != revision || catalogMetadata.revision != build) { + throw new GradleException("Found new jdk version. Please update version.yml to ${catalogMetadata.version} build ${catalogMetadata.revision}") + } + } + private String createElasticCatalogDownloadUrl() { // Ask details to catalog https://jvm-catalog.elastic.co/jdk and return the url to download the JDK - // arch x86_64 never used, only aarch64 if macos - def url = "https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_${major}_${osName}" + // arch x86_64 is default, aarch64 if macos or linux + def url = "https://jvm-catalog.elastic.co/jdk/adoptiumjdk-${revision}+${build}-${osName}" - // Append the cpu's arch only if Mac on aarch64, all the other OSes doesn't have CPU extension + // Append the cpu's arch only if not x86_64, which is the default if (arch == "aarch64") { - url += "_${arch}" + url += "-${arch}" } println "Retrieving JDK from catalog..." def catalogMetadataUrl = URI.create(url).toURL() def catalogConnection = catalogMetadataUrl.openConnection() catalogConnection.requestMethod = 'GET' - assert catalogConnection.responseCode == 200 + if (catalogConnection.responseCode != 200) { + println "Can't find adoptiumjdk ${revision} for ${osName} on Elastic JVM catalog" + throw new GradleException("JVM not present on catalog") + } def metadataRetrieved = catalogConnection.content.text println "Retrieved!" def catalogMetadata = new JsonSlurper().parseText(metadataRetrieved) + validateMetadata(catalogMetadata) + return catalogMetadata.url } - private String createAdoptDownloadUrl() { - String releaseName = major > 8 ? - "jdk-${revision}+${build}" : - "jdk${revision}u${build}" - String vendorOsName = vendorOsName(osName) - switch (vendor) { - case "adoptium": - return "https://api.adoptium.net/v3/binary/version/${releaseName}/${vendorOsName}/${arch}/jdk/hotspot/normal/adoptium" - default: - throw RuntimeException("Can't handle vendor: ${vendor}") + //Verify that the artifact metadata correspond to the request, if not throws an error + private void validateMetadata(Map metadata) { + if (metadata.version != revision) { + throw new GradleException("Expected to retrieve a JDK for version ${revision} but received: ${metadata.version}") + } + if (!isSameArchitecture(metadata.architecture)) { + throw new GradleException("Expected to retrieve a JDK for architecture ${arch} but received: ${metadata.architecture}") } } - private String vendorOsName(String osName) { - if (osName == "darwin") - return "mac" - return osName + private boolean isSameArchitecture(String metadataArch) { + if (arch == 'x64') { + return metadataArch == 'x86_64' + } + return metadata.architecture == arch } private String parseJdkArchitecture(String jdkArch) { @@ -791,7 +811,7 @@ class JDKDetails { return "aarch64" break default: - throw RuntimeException("Can't handle CPU architechture: ${jdkArch}") + throw new GradleException("Can't handle CPU architechture: ${jdkArch}") } } } @@ -840,6 +860,15 @@ tasks.register("downloadJdk", Download) { } } +tasks.register("checkNewJdkVersion") { + def versionYml = new Yaml().load(new File("$projectDir/versions.yml").text) + + // use Linux x86_64 as canary platform + def jdkDetails = new JDKDetails(versionYml, "linux", "x86_64") + // throws Gradle exception if local and remote doesn't match + jdkDetails.checkLocalVersionMatchingLatest() +} + tasks.register("deleteLocalJdk", Delete) { // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] String osName = selectOsType() diff --git a/catalog-info.yaml b/catalog-info.yaml index 8b3608c454e..ad928a7d4b7 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -33,6 +33,7 @@ spec: - resource:logstash-windows-jdk-matrix-pipeline - resource:logstash-benchmark-pipeline - resource:logstash-health-report-tests-pipeline + - resource:logstash-jdk-availability-check-pipeline # *********************************** # Declare serverless IT pipeline @@ -572,6 +573,12 @@ spec: env: PIPELINES_TO_TRIGGER: 'logstash-exhaustive-tests-pipeline' EXCLUDE_BRANCHES: 'main' + JDK availability check: + branch: main + cronline: 0 2 * * 1 # every Monday@2AM UTC + message: Weekly trigger of JDK update availability pipeline per branch + env: + PIPELINES_TO_TRIGGER: 'logstash-jdk-availability-check-pipeline' skip_intermediate_builds: true provider_settings: trigger_mode: none @@ -745,4 +752,46 @@ spec: # ******************************* # SECTION END: Health Report Tests pipeline +# ******************************* + +# *********************************** +# Declare JDK check pipeline +# *********************************** +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: logstash-jdk-availability-check-pipeline + description: ":logstash: check availability of new JDK version" +spec: + type: buildkite-pipeline + owner: group:logstash + system: platform-ingest + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: logstash-jdk-availability-check-pipeline + spec: + repository: elastic/logstash + pipeline_file: ".buildkite/jdk_availability_check_pipeline.yml" + maximum_timeout_in_minutes: 10 + provider_settings: + trigger_mode: none # don't trigger jobs from github activity + env: + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build' + SLACK_NOTIFICATIONS_ON_SUCCESS: 'false' + SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true' + teams: + logstash: + access_level: MANAGE_BUILD_AND_READ + ingest-eng-prod: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +# ******************************* +# SECTION END: JDK check pipeline # ******************************* \ No newline at end of file diff --git a/ci/check_jdk_version_availability.sh b/ci/check_jdk_version_availability.sh new file mode 100755 index 00000000000..2ce40dc7b2f --- /dev/null +++ b/ci/check_jdk_version_availability.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -eo pipefail + +export GRADLE_OPTS="-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" + +echo "Checking local JDK version against latest remote from JVM catalog" +./gradlew checkNewJdkVersion \ No newline at end of file