diff --git a/.github/workflows/publish-java.yml b/.github/workflows/publish-java.yml index 0b6de429..4a2c1d95 100644 --- a/.github/workflows/publish-java.yml +++ b/.github/workflows/publish-java.yml @@ -14,6 +14,22 @@ jobs: steps: - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # pin@v4 + - name: Set up Java version + run: | + declare -A map_java_version + map_java_version=( + ["java8al2"]="8" + ["java11"]="11" + ["java17"]="17" + ["java21"]="21" + ) + java_numeric_version=${map_java_version[${{ matrix.java-version }}]} + echo "JAVA_NUMERIC_VERSION=$java_numeric_version" >> $GITHUB_ENV + - name: Use Java ${{ env.JAVA_NUMERIC_VERSION }} + uses: actions/setup-java@v4 + with: + distribution: 'corretto' + java-version: ${{ env.JAVA_NUMERIC_VERSION }} - name: Check Tag id: java-check-tag run: | @@ -26,3 +42,11 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: make publish-${{ matrix.java-version }}-ci + - name: Publish ECR image for ${{ matrix.java-version }} + if: steps.java-check-tag.outputs.match == 'true' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + cd java + ./publish-layers.sh build-publish-${{ matrix.java-version }}-ecr-image diff --git a/.github/workflows/publish-node.yml b/.github/workflows/publish-node.yml index 0c071a0d..b905af26 100644 --- a/.github/workflows/publish-node.yml +++ b/.github/workflows/publish-node.yml @@ -32,6 +32,14 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: make publish-nodejs${{ matrix.node-version }}x-ci + - name: Publish ECR image for nodejs${{ matrix.node-version }} + if: steps.node-check-tag.outputs.match == 'true' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + cd nodejs + ./publish-layers.sh build-publish-nodejs${{ matrix.node-version }}x-ecr-image - name: Upload Unit Test Coverage if: steps.node-check-tag.outputs.match == 'true' uses: codecov/codecov-action@v3 diff --git a/.github/workflows/publish-ruby.yml b/.github/workflows/publish-ruby.yml index 624a8802..d53d753d 100644 --- a/.github/workflows/publish-ruby.yml +++ b/.github/workflows/publish-ruby.yml @@ -38,3 +38,11 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: make publish-ruby${{ steps.ruby-version-without-dot.outputs.VERSION }}-ci + - name: Publish ECR image for ruby ${{ matrix.ruby-version }} + if: steps.ruby-check-tag.outputs.match == 'true' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + cd ruby + ./publish-ecr-images.sh ruby${{ matrix.ruby-version }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 2a7ba19e..a97b9408 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.7', '3.8', '3.9'] steps: - uses: actions/checkout@v3 diff --git a/Dockerfile.ecrImage b/Dockerfile.ecrImage new file mode 100644 index 00000000..816defa0 --- /dev/null +++ b/Dockerfile.ecrImage @@ -0,0 +1,13 @@ +FROM alpine:latest + +ARG layer_zip +ARG file_without_dist + +RUN apk update && apk add --no-cache curl unzip + +WORKDIR / + +COPY ${layer_zip} . + +RUN unzip ${file_without_dist} -d ./opt +RUN rm ${file_without_dist} diff --git a/java/publish-layers.sh b/java/publish-layers.sh index 102aaea4..b139ef9f 100755 --- a/java/publish-layers.sh +++ b/java/publish-layers.sh @@ -204,6 +204,30 @@ case "$1" in publish-java21-arm64 publish-java21-x86 ;; +"build-publish-java8al2-ecr-image") + build-java8al2-arm64 + publish_docker_ecr $JAVA8_DIST_ARM64 java8 arm64 + build-java8al2-x86 + publish_docker_ecr $JAVA8_DIST_X86_64 java8 x86_64 + ;; +"build-publish-java11-ecr-image") + build-java11-arm64 + publish_docker_ecr $JAVA11_DIST_ARM64 java11 arm64 + build-java11-x86 + publish_docker_ecr $JAVA11_DIST_X86_64 java11 x86_64 + ;; +"build-publish-java17-ecr-image") + build-java17-arm64 + publish_docker_ecr $JAVA17_DIST_ARM64 java17 arm64 + build-java17-x86 + publish_docker_ecr $JAVA17_DIST_X86_64 java17 x86_64 + ;; +"build-publish-java21-ecr-image") + build-java21-arm64 + publish_docker_ecr $JAVA21_DIST_ARM64 java21 arm64 + build-java21-x86 + publish_docker_ecr $JAVA21_DIST_X86_64 java21 x86_64 + ;; "java8al2") $0 build-java8al2 $0 publish-java8al2 diff --git a/libBuild.sh b/libBuild.sh index f11f1a1c..484050c7 100644 --- a/libBuild.sh +++ b/libBuild.sh @@ -324,3 +324,57 @@ function publish_docker_ecr { } + +function publish_docker_ecr { + layer_archive=$1 + runtime_name=$2 + arch=$3 + + if [[ ${arch} =~ 'arm64' ]]; + then arch_flag="-arm64" + else arch_flag="" + fi + + version_flag=$(echo "$runtime_name" | sed 's/[^0-9]//g') + language_flag=$(echo "$runtime_name" | sed 's/[0-9].*//') + + + # Remove 'dist/' prefix + if [[ $layer_archive == dist/* ]]; then + file_without_dist="${layer_archive#dist/}" + echo "File without 'dist/': $file_without_dist" + else + file_without_dist=$layer_archive + echo "File does not start with 'dist/': $file_without_dist" + fi + + # public ecr repository name + # maintainer can use this("q6k3q1g1") repo name for testing + repository="x6n7b2o2" + + # copy dockerfile + cp ../Dockerfile.ecrImage . + + echo "Running : aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/${repository}" + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/${repository} + + echo "docker build -t layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest \ + -f Dockerfile.ecrImage \ + --build-arg layer_zip=${layer_archive} \ + --build-arg file_without_dist=${file_without_dist} \ + ." + + docker build -t layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest \ + -f Dockerfile.ecrImage \ + --build-arg layer_zip=${layer_archive} \ + --build-arg file_without_dist=${file_without_dist} \ + . + + echo "docker tag layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}" + docker tag layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag} + echo "docker push public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}" + docker push public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag} + + # delete dockerfile + rm -rf Dockerfile.ecrImage +} diff --git a/nodejs/publish-layers.sh b/nodejs/publish-layers.sh index 419188c6..e4680a1f 100755 --- a/nodejs/publish-layers.sh +++ b/nodejs/publish-layers.sh @@ -188,6 +188,24 @@ case "$1" in publish-nodejs20x-arm64 publish-nodejs20x-x86 ;; +"build-publish-nodejs16x-ecr-image") + build-nodejs16x-arm64 + publish_docker_ecr $NJS16X_DIST_ARM64 nodejs16.x arm64 + build-nodejs16x-x86 + publish_docker_ecr $NJS16X_DIST_X86_64 nodejs16.x x86_64 + ;; +"build-publish-nodejs18x-ecr-image") + build-nodejs18x-arm64 + publish_docker_ecr $NJS18X_DIST_ARM64 nodejs18.x arm64 + build-nodejs18x-x86 + publish_docker_ecr $NJS18X_DIST_X86_64 nodejs18.x x86_64 + ;; +"build-publish-nodejs20x-ecr-image") + build-nodejs20x-arm64 + publish_docker_ecr $NJS20X_DIST_ARM64 nodejs20.x arm64 + build-nodejs20x-x86 + publish_docker_ecr $NJS20X_DIST_X86_64 nodejs20.x x86_64 + ;; "nodejs16x") $0 build-nodejs16x $0 publish-nodejs16x diff --git a/python/publish-layers.sh b/python/publish-layers.sh index 1c21974a..d294d984 100755 --- a/python/publish-layers.sh +++ b/python/publish-layers.sh @@ -288,40 +288,52 @@ function publish-python312-x86 { publish_layer $PY312_DIST_X86_64 $region python3.12 x86_64 done } + case "$1" in "python3.7") build-python37-x86 publish-python37-x86 + publish_docker_ecr $PY37_DIST_X86_64 python3.7 x86_64 ;; "python3.8") build-python38-arm64 publish-python38-arm64 + publish_docker_ecr $PY38_DIST_ARM64 python3.8 arm64 build-python38-x86 publish-python38-x86 + publish_docker_ecr $PY38_DIST_X86_64 python3.8 x86_64 ;; "python3.9") build-python39-arm64 publish-python39-arm64 + publish_docker_ecr $PY39_DIST_ARM64 python3.9 arm64 build-python39-x86 publish-python39-x86 + publish_docker_ecr $PY39_DIST_X86_64 python3.9 x86_64 ;; "python3.10") build-python310-arm64 publish-python310-arm64 + publish_docker_ecr $PY310_DIST_ARM64 python3.10 arm64 build-python310-x86 publish-python310-x86 + publish_docker_ecr $PY310_DIST_X86_64 python3.10 x86_64 ;; "python3.11") build-python311-arm64 publish-python311-arm64 + publish_docker_ecr $PY311_DIST_ARM64 python3.11 arm64 build-python311-x86 publish-python311-x86 + publish_docker_ecr $PY311_DIST_X86_64 python3.11 x86_64 ;; "python3.12") build-python312-arm64 publish-python312-arm64 + publish_docker_ecr $PY312_DIST_ARM64 python3.12 arm64 build-python312-x86 publish-python312-x86 + publish_docker_ecr $PY312_DIST_X86_64 python3.12 x86_64 ;; *) usage diff --git a/ruby/publish-ecr-images.sh b/ruby/publish-ecr-images.sh new file mode 100755 index 00000000..e3b8a778 --- /dev/null +++ b/ruby/publish-ecr-images.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +# This script creates an AWS Lambda Ruby layer .zip file for each supported +# architecture. The Ruby content does not change between architectures, but +# the included Go based AWS Lambda extension does. The .zip files are written +# to dist/ and from there they are uploaded to AWS via ../libBuild.sh +# functionality. +# +# Each .zip file is structured like so for Ruby: +# ruby/gems/.0 -- AWS sets this as GEM_PATH. It's where the agent lives +# ruby/lib -- AWS sets this as RUBYLIB. It's where the wrapper script lives +# extensions/ -- Where the NR Go based extension content lives +# preview-extensions-* -- Extensions preview file + +RUBY_DIR=ruby +DIST_DIR=dist +WRAPPER_FILE=newrelic_lambda_wrapper.rb +# Set this to a path to a clone of newrelic-lambda-extension to build +# an extension from scratch instead of downloading one. Set the path to '' +# to simply download a prebuilt one. +# EXTENSION_CLONE_PATH='../../newrelic-lambda-extension_fallwith' +EXTENSION_CLONE_PATH='' + +source ../libBuild.sh + +function usage { + echo "./publish-ecr-images.sh [ruby3.2|ruby3.3]" +} + +RUBY33_DIST_ARM64=$DIST_DIR/ruby33.arm64.zip +RUBY32_DIST_ARM64=$DIST_DIR/ruby32.arm64.zip + +RUBY33_DIST_X86_64=$DIST_DIR/ruby33.x86_64.zip +RUBY32_DIST_X86_64=$DIST_DIR/ruby32.x86_64.zip + + +function build_and_publish_ruby_for_arch { + local dist_file=$1 + local ruby_version=$2 + local arch=$3 + echo "Building New Relic layer for ruby v$ruby_version ($arch)" + + + rm -rf $RUBY_DIR $dist_file + mkdir -p $DIST_DIR + + bundle config set --local without development + bundle config set --local path . # Bundler will create a 'ruby' dir beneath here + bundle install + + local base_dir="$RUBY_DIR/gems/$ruby_version.0" + + # Bundler will have created ./ruby//gems + # AWS wants ./ruby/gems/ + # So we need to flip the directory structure around and also use the right + # Ruby version. For building, we insist on the same major Ruby version but + # are relaxed on the minor version. + mkdir $RUBY_DIR/gems + + # allow Ruby versions other than the target version to bundle + # so if Bundler used Ruby v3.3 and the target is v3.2, the '3.3.0' dir + # gets renamed to '3.2.0' + mv $RUBY_DIR/${ruby_version:0:1}* $base_dir + + for sub_dir in 'bin' 'build_info' 'cache' 'doc' 'extensions' 'plugins'; do + rm -rf $base_dir/$sub_dir + done + + mkdir -p $RUBY_DIR/lib + cp $WRAPPER_FILE $RUBY_DIR/lib + + # if Gemfile points Bundler to GitHub for the agent, discard extraneous repo + # content and repackage the vendored gem content as if it were sourced + # from RubyGems.org + if [[ -e "$base_dir/bundler" ]]; then + local phony_version=$(date +'%s') + mkdir -p $base_dir/gems # dir will exist if non-agent, non-dev gems are in Gemfile + local nr_dir=$base_dir/gems/newrelic_rpm-$phony_version + mv $base_dir/bundler/gems/newrelic-ruby-agent* $nr_dir + rm -rf $base_dir/bundler + mkdir $base_dir/specifications + echo -e "Gem::Specification.new {|s| s.name = 'newrelic_rpm'; s.version = '$phony_version'}" > $base_dir/specifications/newrelic_rpm-$phony_version.gemspec + for sub_dir in '.git' '.github' '.gitignore' '.rubocop.yml' '.rubocop_todo.yml' '.simplecov' '.snyk' '.yardopts' 'Brewfile' 'config' 'CONTRIBUTING.md' 'docker-compose.yml' 'DOCKER.md' 'Dockerfile' 'Gemfile' 'Guardfile' 'infinite_tracing' 'init.rb' 'install.rb' 'lefthook.yml' 'newrelic.yml' 'README.md' 'test' 'THIRD_PARTY_NOTICES.md' 'Thorfile' 'recipes' '.build_ignore'; do + rm -rf "$nr_dir/$sub_dir" + done fi + + if [ "$EXTENSION_CLONE_PATH" == "" ]; then + echo "Downloading prebuilt extension..." + download_extension $arch + else + echo "Building an extension from a local clone..." + here=$PWD + cd "$EXTENSION_CLONE_PATH" + make "dist-$arch" + mv "$EXTENSION_DIST_DIR" "$here/$EXTENSION_DIST_DIR" + mv "$EXTENSION_DIST_PREVIEW_FILE" "$here/$EXTENSION_DIST_PREVIEW_FILE" + cd $here + fi + + zip -rq $dist_file $RUBY_DIR $EXTENSION_DIST_DIR $EXTENSION_DIST_PREVIEW_FILE + rm -rf $RUBY_DIR $EXTENSION_DIST_DIR $EXTENSION_DIST_PREVIEW_FILE + echo "Build complete: ${dist_file}" +} + +set +u # permit $1 to be unbound so that '*' matches it when no args are present +case "$1" in + "ruby3.3") + build_and_publish_ruby_for_arch $RUBY33_DIST_X86_64 '3.3' 'x86_64' + publish_docker_ecr $RUBY33_DIST_X86_64 ruby3.3 x86_64 + build_and_publish_ruby_for_arch $RUBY33_DIST_ARM64 '3.3' 'arm64' + publish_docker_ecr $RUBY33_DIST_ARM64 ruby3.3 arm64 + + ;; + "ruby3.2") + build_and_publish_ruby_for_arch $RUBY32_DIST_X86_64 '3.2' 'x86_64' + publish_docker_ecr $RUBY32_DIST_X86_64 ruby3.2 x86_64 + build_and_publish_ruby_for_arch $RUBY32_DIST_ARM64 '3.2' 'arm64' + publish_docker_ecr $RUBY32_DIST_ARM64 ruby3.2 arm64 + ;; + *) + usage + ;; +esac