diff --git a/.github/workflows/publish-java.yml b/.github/workflows/publish-java.yml index 0b6de429..4a2c1d95 100644 --- a/.github/workflows/publish-java.yml +++ b/.github/workflows/publish-java.yml @@ -14,6 +14,22 @@ jobs: steps: - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # pin@v4 + - name: Set up Java version + run: | + declare -A map_java_version + map_java_version=( + ["java8al2"]="8" + ["java11"]="11" + ["java17"]="17" + ["java21"]="21" + ) + java_numeric_version=${map_java_version[${{ matrix.java-version }}]} + echo "JAVA_NUMERIC_VERSION=$java_numeric_version" >> $GITHUB_ENV + - name: Use Java ${{ env.JAVA_NUMERIC_VERSION }} + uses: actions/setup-java@v4 + with: + distribution: 'corretto' + java-version: ${{ env.JAVA_NUMERIC_VERSION }} - name: Check Tag id: java-check-tag run: | @@ -26,3 +42,11 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: make publish-${{ matrix.java-version }}-ci + - name: Publish ECR image for ${{ matrix.java-version }} + if: steps.java-check-tag.outputs.match == 'true' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + cd java + ./publish-layers.sh build-publish-${{ matrix.java-version }}-ecr-image diff --git a/.github/workflows/publish-node.yml b/.github/workflows/publish-node.yml index 0c071a0d..b905af26 100644 --- a/.github/workflows/publish-node.yml +++ b/.github/workflows/publish-node.yml @@ -32,6 +32,14 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: make publish-nodejs${{ matrix.node-version }}x-ci + - name: Publish ECR image for nodejs${{ matrix.node-version }} + if: steps.node-check-tag.outputs.match == 'true' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + cd nodejs + ./publish-layers.sh build-publish-nodejs${{ matrix.node-version }}x-ecr-image - name: Upload Unit Test Coverage if: steps.node-check-tag.outputs.match == 'true' uses: codecov/codecov-action@v3 diff --git a/.github/workflows/publish-ruby.yml b/.github/workflows/publish-ruby.yml index 624a8802..d53d753d 100644 --- a/.github/workflows/publish-ruby.yml +++ b/.github/workflows/publish-ruby.yml @@ -38,3 +38,11 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: make publish-ruby${{ steps.ruby-version-without-dot.outputs.VERSION }}-ci + - name: Publish ECR image for ruby ${{ matrix.ruby-version }} + if: steps.ruby-check-tag.outputs.match == 'true' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + cd ruby + ./publish-ecr-images.sh ruby${{ matrix.ruby-version }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 2a7ba19e..a97b9408 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.7', '3.8', '3.9'] steps: - uses: actions/checkout@v3 diff --git a/Dockerfile.ecrImage b/Dockerfile.ecrImage new file mode 100644 index 00000000..816defa0 --- /dev/null +++ b/Dockerfile.ecrImage @@ -0,0 +1,13 @@ +FROM alpine:latest + +ARG layer_zip +ARG file_without_dist + +RUN apk update && apk add --no-cache curl unzip + +WORKDIR / + +COPY ${layer_zip} . + +RUN unzip ${file_without_dist} -d ./opt +RUN rm ${file_without_dist} diff --git a/README.md b/README.md index ab82d829..b3cfd5a6 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repository contains source code and utilities to build and publish New Relic's public AWS Lambda layers. -Most users should use our published layers which are chosen automatically via the [CLI tool](https://github.com/newrelic/newrelic-lambda-cli). Those layers are published to be public and are available [here](https://nr-layers.iopipe.com). +Most users should use our published layers which are chosen automatically via the [CLI tool](https://github.com/newrelic/newrelic-lambda-cli). Those layers are published to be public and are available [here](https://layers.newrelic-external.com/). This tool is released for users seeking to deploy their own copies of the New Relic Lambda Layers into their accounts, or to modify and publish their own customized wrapper layers. diff --git a/java/publish-layers.sh b/java/publish-layers.sh index 102aaea4..b139ef9f 100755 --- a/java/publish-layers.sh +++ b/java/publish-layers.sh @@ -204,6 +204,30 @@ case "$1" in publish-java21-arm64 publish-java21-x86 ;; +"build-publish-java8al2-ecr-image") + build-java8al2-arm64 + publish_docker_ecr $JAVA8_DIST_ARM64 java8 arm64 + build-java8al2-x86 + publish_docker_ecr $JAVA8_DIST_X86_64 java8 x86_64 + ;; +"build-publish-java11-ecr-image") + build-java11-arm64 + publish_docker_ecr $JAVA11_DIST_ARM64 java11 arm64 + build-java11-x86 + publish_docker_ecr $JAVA11_DIST_X86_64 java11 x86_64 + ;; +"build-publish-java17-ecr-image") + build-java17-arm64 + publish_docker_ecr $JAVA17_DIST_ARM64 java17 arm64 + build-java17-x86 + publish_docker_ecr $JAVA17_DIST_X86_64 java17 x86_64 + ;; +"build-publish-java21-ecr-image") + build-java21-arm64 + publish_docker_ecr $JAVA21_DIST_ARM64 java21 arm64 + build-java21-x86 + publish_docker_ecr $JAVA21_DIST_X86_64 java21 x86_64 + ;; "java8al2") $0 build-java8al2 $0 publish-java8al2 diff --git a/libBuild.sh b/libBuild.sh index 49a2ddbd..484050c7 100644 --- a/libBuild.sh +++ b/libBuild.sh @@ -222,7 +222,7 @@ function publish_layer { compat_list=( $runtime_name ) if [[ $runtime_name == "provided" ]] - then compat_list=("provided" "provided.al2" "provided.al2023" "dotnetcore3.1" "dotnet6") + then compat_list=("provided" "provided.al2" "provided.al2023" "dotnetcore3.1") fi if [[ $runtime_name == "dotnet" ]] @@ -324,3 +324,57 @@ function publish_docker_ecr { } + +function publish_docker_ecr { + layer_archive=$1 + runtime_name=$2 + arch=$3 + + if [[ ${arch} =~ 'arm64' ]]; + then arch_flag="-arm64" + else arch_flag="" + fi + + version_flag=$(echo "$runtime_name" | sed 's/[^0-9]//g') + language_flag=$(echo "$runtime_name" | sed 's/[0-9].*//') + + + # Remove 'dist/' prefix + if [[ $layer_archive == dist/* ]]; then + file_without_dist="${layer_archive#dist/}" + echo "File without 'dist/': $file_without_dist" + else + file_without_dist=$layer_archive + echo "File does not start with 'dist/': $file_without_dist" + fi + + # public ecr repository name + # maintainer can use this("q6k3q1g1") repo name for testing + repository="x6n7b2o2" + + # copy dockerfile + cp ../Dockerfile.ecrImage . + + echo "Running : aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/${repository}" + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/${repository} + + echo "docker build -t layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest \ + -f Dockerfile.ecrImage \ + --build-arg layer_zip=${layer_archive} \ + --build-arg file_without_dist=${file_without_dist} \ + ." + + docker build -t layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest \ + -f Dockerfile.ecrImage \ + --build-arg layer_zip=${layer_archive} \ + --build-arg file_without_dist=${file_without_dist} \ + . + + echo "docker tag layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}" + docker tag layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag} + echo "docker push public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}" + docker push public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag} + + # delete dockerfile + rm -rf Dockerfile.ecrImage +} diff --git a/nodejs/publish-layers.sh b/nodejs/publish-layers.sh index 419188c6..e4680a1f 100755 --- a/nodejs/publish-layers.sh +++ b/nodejs/publish-layers.sh @@ -188,6 +188,24 @@ case "$1" in publish-nodejs20x-arm64 publish-nodejs20x-x86 ;; +"build-publish-nodejs16x-ecr-image") + build-nodejs16x-arm64 + publish_docker_ecr $NJS16X_DIST_ARM64 nodejs16.x arm64 + build-nodejs16x-x86 + publish_docker_ecr $NJS16X_DIST_X86_64 nodejs16.x x86_64 + ;; +"build-publish-nodejs18x-ecr-image") + build-nodejs18x-arm64 + publish_docker_ecr $NJS18X_DIST_ARM64 nodejs18.x arm64 + build-nodejs18x-x86 + publish_docker_ecr $NJS18X_DIST_X86_64 nodejs18.x x86_64 + ;; +"build-publish-nodejs20x-ecr-image") + build-nodejs20x-arm64 + publish_docker_ecr $NJS20X_DIST_ARM64 nodejs20.x arm64 + build-nodejs20x-x86 + publish_docker_ecr $NJS20X_DIST_X86_64 nodejs20.x x86_64 + ;; "nodejs16x") $0 build-nodejs16x $0 publish-nodejs16x diff --git a/python/publish-layers.sh b/python/publish-layers.sh index 1c21974a..d294d984 100755 --- a/python/publish-layers.sh +++ b/python/publish-layers.sh @@ -288,40 +288,52 @@ function publish-python312-x86 { publish_layer $PY312_DIST_X86_64 $region python3.12 x86_64 done } + case "$1" in "python3.7") build-python37-x86 publish-python37-x86 + publish_docker_ecr $PY37_DIST_X86_64 python3.7 x86_64 ;; "python3.8") build-python38-arm64 publish-python38-arm64 + publish_docker_ecr $PY38_DIST_ARM64 python3.8 arm64 build-python38-x86 publish-python38-x86 + publish_docker_ecr $PY38_DIST_X86_64 python3.8 x86_64 ;; "python3.9") build-python39-arm64 publish-python39-arm64 + publish_docker_ecr $PY39_DIST_ARM64 python3.9 arm64 build-python39-x86 publish-python39-x86 + publish_docker_ecr $PY39_DIST_X86_64 python3.9 x86_64 ;; "python3.10") build-python310-arm64 publish-python310-arm64 + publish_docker_ecr $PY310_DIST_ARM64 python3.10 arm64 build-python310-x86 publish-python310-x86 + publish_docker_ecr $PY310_DIST_X86_64 python3.10 x86_64 ;; "python3.11") build-python311-arm64 publish-python311-arm64 + publish_docker_ecr $PY311_DIST_ARM64 python3.11 arm64 build-python311-x86 publish-python311-x86 + publish_docker_ecr $PY311_DIST_X86_64 python3.11 x86_64 ;; "python3.12") build-python312-arm64 publish-python312-arm64 + publish_docker_ecr $PY312_DIST_ARM64 python3.12 arm64 build-python312-x86 publish-python312-x86 + publish_docker_ecr $PY312_DIST_X86_64 python3.12 x86_64 ;; *) usage diff --git a/ruby/publish-ecr-images.sh b/ruby/publish-ecr-images.sh new file mode 100755 index 00000000..e3b8a778 --- /dev/null +++ b/ruby/publish-ecr-images.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +# This script creates an AWS Lambda Ruby layer .zip file for each supported +# architecture. The Ruby content does not change between architectures, but +# the included Go based AWS Lambda extension does. The .zip files are written +# to dist/ and from there they are uploaded to AWS via ../libBuild.sh +# functionality. +# +# Each .zip file is structured like so for Ruby: +# ruby/gems/.0 -- AWS sets this as GEM_PATH. It's where the agent lives +# ruby/lib -- AWS sets this as RUBYLIB. It's where the wrapper script lives +# extensions/ -- Where the NR Go based extension content lives +# preview-extensions-* -- Extensions preview file + +RUBY_DIR=ruby +DIST_DIR=dist +WRAPPER_FILE=newrelic_lambda_wrapper.rb +# Set this to a path to a clone of newrelic-lambda-extension to build +# an extension from scratch instead of downloading one. Set the path to '' +# to simply download a prebuilt one. +# EXTENSION_CLONE_PATH='../../newrelic-lambda-extension_fallwith' +EXTENSION_CLONE_PATH='' + +source ../libBuild.sh + +function usage { + echo "./publish-ecr-images.sh [ruby3.2|ruby3.3]" +} + +RUBY33_DIST_ARM64=$DIST_DIR/ruby33.arm64.zip +RUBY32_DIST_ARM64=$DIST_DIR/ruby32.arm64.zip + +RUBY33_DIST_X86_64=$DIST_DIR/ruby33.x86_64.zip +RUBY32_DIST_X86_64=$DIST_DIR/ruby32.x86_64.zip + + +function build_and_publish_ruby_for_arch { + local dist_file=$1 + local ruby_version=$2 + local arch=$3 + echo "Building New Relic layer for ruby v$ruby_version ($arch)" + + + rm -rf $RUBY_DIR $dist_file + mkdir -p $DIST_DIR + + bundle config set --local without development + bundle config set --local path . # Bundler will create a 'ruby' dir beneath here + bundle install + + local base_dir="$RUBY_DIR/gems/$ruby_version.0" + + # Bundler will have created ./ruby//gems + # AWS wants ./ruby/gems/ + # So we need to flip the directory structure around and also use the right + # Ruby version. For building, we insist on the same major Ruby version but + # are relaxed on the minor version. + mkdir $RUBY_DIR/gems + + # allow Ruby versions other than the target version to bundle + # so if Bundler used Ruby v3.3 and the target is v3.2, the '3.3.0' dir + # gets renamed to '3.2.0' + mv $RUBY_DIR/${ruby_version:0:1}* $base_dir + + for sub_dir in 'bin' 'build_info' 'cache' 'doc' 'extensions' 'plugins'; do + rm -rf $base_dir/$sub_dir + done + + mkdir -p $RUBY_DIR/lib + cp $WRAPPER_FILE $RUBY_DIR/lib + + # if Gemfile points Bundler to GitHub for the agent, discard extraneous repo + # content and repackage the vendored gem content as if it were sourced + # from RubyGems.org + if [[ -e "$base_dir/bundler" ]]; then + local phony_version=$(date +'%s') + mkdir -p $base_dir/gems # dir will exist if non-agent, non-dev gems are in Gemfile + local nr_dir=$base_dir/gems/newrelic_rpm-$phony_version + mv $base_dir/bundler/gems/newrelic-ruby-agent* $nr_dir + rm -rf $base_dir/bundler + mkdir $base_dir/specifications + echo -e "Gem::Specification.new {|s| s.name = 'newrelic_rpm'; s.version = '$phony_version'}" > $base_dir/specifications/newrelic_rpm-$phony_version.gemspec + for sub_dir in '.git' '.github' '.gitignore' '.rubocop.yml' '.rubocop_todo.yml' '.simplecov' '.snyk' '.yardopts' 'Brewfile' 'config' 'CONTRIBUTING.md' 'docker-compose.yml' 'DOCKER.md' 'Dockerfile' 'Gemfile' 'Guardfile' 'infinite_tracing' 'init.rb' 'install.rb' 'lefthook.yml' 'newrelic.yml' 'README.md' 'test' 'THIRD_PARTY_NOTICES.md' 'Thorfile' 'recipes' '.build_ignore'; do + rm -rf "$nr_dir/$sub_dir" + done fi + + if [ "$EXTENSION_CLONE_PATH" == "" ]; then + echo "Downloading prebuilt extension..." + download_extension $arch + else + echo "Building an extension from a local clone..." + here=$PWD + cd "$EXTENSION_CLONE_PATH" + make "dist-$arch" + mv "$EXTENSION_DIST_DIR" "$here/$EXTENSION_DIST_DIR" + mv "$EXTENSION_DIST_PREVIEW_FILE" "$here/$EXTENSION_DIST_PREVIEW_FILE" + cd $here + fi + + zip -rq $dist_file $RUBY_DIR $EXTENSION_DIST_DIR $EXTENSION_DIST_PREVIEW_FILE + rm -rf $RUBY_DIR $EXTENSION_DIST_DIR $EXTENSION_DIST_PREVIEW_FILE + echo "Build complete: ${dist_file}" +} + +set +u # permit $1 to be unbound so that '*' matches it when no args are present +case "$1" in + "ruby3.3") + build_and_publish_ruby_for_arch $RUBY33_DIST_X86_64 '3.3' 'x86_64' + publish_docker_ecr $RUBY33_DIST_X86_64 ruby3.3 x86_64 + build_and_publish_ruby_for_arch $RUBY33_DIST_ARM64 '3.3' 'arm64' + publish_docker_ecr $RUBY33_DIST_ARM64 ruby3.3 arm64 + + ;; + "ruby3.2") + build_and_publish_ruby_for_arch $RUBY32_DIST_X86_64 '3.2' 'x86_64' + publish_docker_ecr $RUBY32_DIST_X86_64 ruby3.2 x86_64 + build_and_publish_ruby_for_arch $RUBY32_DIST_ARM64 '3.2' 'arm64' + publish_docker_ecr $RUBY32_DIST_ARM64 ruby3.2 arm64 + ;; + *) + usage + ;; +esac