Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master'
Browse files Browse the repository at this point in the history
  • Loading branch information
chaudharysaket committed May 22, 2024
2 parents cfbb8f7 + dac6e90 commit ec5d3e0
Show file tree
Hide file tree
Showing 11 changed files with 289 additions and 3 deletions.
24 changes: 24 additions & 0 deletions .github/workflows/publish-java.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,22 @@ jobs:

steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # pin@v4
- name: Set up Java version
run: |
declare -A map_java_version
map_java_version=(
["java8al2"]="8"
["java11"]="11"
["java17"]="17"
["java21"]="21"
)
java_numeric_version=${map_java_version[${{ matrix.java-version }}]}
echo "JAVA_NUMERIC_VERSION=$java_numeric_version" >> $GITHUB_ENV
- name: Use Java ${{ env.JAVA_NUMERIC_VERSION }}
uses: actions/setup-java@v4
with:
distribution: 'corretto'
java-version: ${{ env.JAVA_NUMERIC_VERSION }}
- name: Check Tag
id: java-check-tag
run: |
Expand All @@ -26,3 +42,11 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: make publish-${{ matrix.java-version }}-ci
- name: Publish ECR image for ${{ matrix.java-version }}
if: steps.java-check-tag.outputs.match == 'true'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
cd java
./publish-layers.sh build-publish-${{ matrix.java-version }}-ecr-image
8 changes: 8 additions & 0 deletions .github/workflows/publish-node.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: make publish-nodejs${{ matrix.node-version }}x-ci
- name: Publish ECR image for nodejs${{ matrix.node-version }}
if: steps.node-check-tag.outputs.match == 'true'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
cd nodejs
./publish-layers.sh build-publish-nodejs${{ matrix.node-version }}x-ecr-image
- name: Upload Unit Test Coverage
if: steps.node-check-tag.outputs.match == 'true'
uses: codecov/codecov-action@v3
Expand Down
8 changes: 8 additions & 0 deletions .github/workflows/publish-ruby.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,11 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: make publish-ruby${{ steps.ruby-version-without-dot.outputs.VERSION }}-ci
- name: Publish ECR image for ruby ${{ matrix.ruby-version }}
if: steps.ruby-check-tag.outputs.match == 'true'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
cd ruby
./publish-ecr-images.sh ruby${{ matrix.ruby-version }}
2 changes: 1 addition & 1 deletion .github/workflows/python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:

strategy:
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
python-version: ['3.7', '3.8', '3.9']
steps:
- uses: actions/checkout@v3

Expand Down
13 changes: 13 additions & 0 deletions Dockerfile.ecrImage
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
FROM alpine:latest

ARG layer_zip
ARG file_without_dist

RUN apk update && apk add --no-cache curl unzip

WORKDIR /

COPY ${layer_zip} .

RUN unzip ${file_without_dist} -d ./opt
RUN rm ${file_without_dist}
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

This repository contains source code and utilities to build and publish New Relic's public AWS Lambda layers.

Most users should use our published layers which are chosen automatically via the [CLI tool](https://github.com/newrelic/newrelic-lambda-cli). Those layers are published to be public and are available [here](https://nr-layers.iopipe.com).
Most users should use our published layers which are chosen automatically via the [CLI tool](https://github.com/newrelic/newrelic-lambda-cli). Those layers are published to be public and are available [here](https://layers.newrelic-external.com/).

This tool is released for users seeking to deploy their own copies of the New Relic Lambda Layers into their accounts, or to modify and publish their own customized wrapper layers.

Expand Down
24 changes: 24 additions & 0 deletions java/publish-layers.sh
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,30 @@ case "$1" in
publish-java21-arm64
publish-java21-x86
;;
"build-publish-java8al2-ecr-image")
build-java8al2-arm64
publish_docker_ecr $JAVA8_DIST_ARM64 java8 arm64
build-java8al2-x86
publish_docker_ecr $JAVA8_DIST_X86_64 java8 x86_64
;;
"build-publish-java11-ecr-image")
build-java11-arm64
publish_docker_ecr $JAVA11_DIST_ARM64 java11 arm64
build-java11-x86
publish_docker_ecr $JAVA11_DIST_X86_64 java11 x86_64
;;
"build-publish-java17-ecr-image")
build-java17-arm64
publish_docker_ecr $JAVA17_DIST_ARM64 java17 arm64
build-java17-x86
publish_docker_ecr $JAVA17_DIST_X86_64 java17 x86_64
;;
"build-publish-java21-ecr-image")
build-java21-arm64
publish_docker_ecr $JAVA21_DIST_ARM64 java21 arm64
build-java21-x86
publish_docker_ecr $JAVA21_DIST_X86_64 java21 x86_64
;;
"java8al2")
$0 build-java8al2
$0 publish-java8al2
Expand Down
56 changes: 55 additions & 1 deletion libBuild.sh
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ function publish_layer {

compat_list=( $runtime_name )
if [[ $runtime_name == "provided" ]]
then compat_list=("provided" "provided.al2" "provided.al2023" "dotnetcore3.1" "dotnet6")
then compat_list=("provided" "provided.al2" "provided.al2023" "dotnetcore3.1")
fi

if [[ $runtime_name == "dotnet" ]]
Expand Down Expand Up @@ -324,3 +324,57 @@ function publish_docker_ecr {


}

function publish_docker_ecr {
layer_archive=$1
runtime_name=$2
arch=$3

if [[ ${arch} =~ 'arm64' ]];
then arch_flag="-arm64"
else arch_flag=""
fi

version_flag=$(echo "$runtime_name" | sed 's/[^0-9]//g')
language_flag=$(echo "$runtime_name" | sed 's/[0-9].*//')


# Remove 'dist/' prefix
if [[ $layer_archive == dist/* ]]; then
file_without_dist="${layer_archive#dist/}"
echo "File without 'dist/': $file_without_dist"
else
file_without_dist=$layer_archive
echo "File does not start with 'dist/': $file_without_dist"
fi

# public ecr repository name
# maintainer can use this("q6k3q1g1") repo name for testing
repository="x6n7b2o2"

# copy dockerfile
cp ../Dockerfile.ecrImage .

echo "Running : aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/${repository}"
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/${repository}

echo "docker build -t layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest \
-f Dockerfile.ecrImage \
--build-arg layer_zip=${layer_archive} \
--build-arg file_without_dist=${file_without_dist} \
."

docker build -t layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest \
-f Dockerfile.ecrImage \
--build-arg layer_zip=${layer_archive} \
--build-arg file_without_dist=${file_without_dist} \
.

echo "docker tag layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}"
docker tag layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}
echo "docker push public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}"
docker push public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}

# delete dockerfile
rm -rf Dockerfile.ecrImage
}
18 changes: 18 additions & 0 deletions nodejs/publish-layers.sh
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,24 @@ case "$1" in
publish-nodejs20x-arm64
publish-nodejs20x-x86
;;
"build-publish-nodejs16x-ecr-image")
build-nodejs16x-arm64
publish_docker_ecr $NJS16X_DIST_ARM64 nodejs16.x arm64
build-nodejs16x-x86
publish_docker_ecr $NJS16X_DIST_X86_64 nodejs16.x x86_64
;;
"build-publish-nodejs18x-ecr-image")
build-nodejs18x-arm64
publish_docker_ecr $NJS18X_DIST_ARM64 nodejs18.x arm64
build-nodejs18x-x86
publish_docker_ecr $NJS18X_DIST_X86_64 nodejs18.x x86_64
;;
"build-publish-nodejs20x-ecr-image")
build-nodejs20x-arm64
publish_docker_ecr $NJS20X_DIST_ARM64 nodejs20.x arm64
build-nodejs20x-x86
publish_docker_ecr $NJS20X_DIST_X86_64 nodejs20.x x86_64
;;
"nodejs16x")
$0 build-nodejs16x
$0 publish-nodejs16x
Expand Down
12 changes: 12 additions & 0 deletions python/publish-layers.sh
Original file line number Diff line number Diff line change
Expand Up @@ -288,40 +288,52 @@ function publish-python312-x86 {
publish_layer $PY312_DIST_X86_64 $region python3.12 x86_64
done
}

case "$1" in
"python3.7")
build-python37-x86
publish-python37-x86
publish_docker_ecr $PY37_DIST_X86_64 python3.7 x86_64
;;
"python3.8")
build-python38-arm64
publish-python38-arm64
publish_docker_ecr $PY38_DIST_ARM64 python3.8 arm64
build-python38-x86
publish-python38-x86
publish_docker_ecr $PY38_DIST_X86_64 python3.8 x86_64
;;
"python3.9")
build-python39-arm64
publish-python39-arm64
publish_docker_ecr $PY39_DIST_ARM64 python3.9 arm64
build-python39-x86
publish-python39-x86
publish_docker_ecr $PY39_DIST_X86_64 python3.9 x86_64
;;
"python3.10")
build-python310-arm64
publish-python310-arm64
publish_docker_ecr $PY310_DIST_ARM64 python3.10 arm64
build-python310-x86
publish-python310-x86
publish_docker_ecr $PY310_DIST_X86_64 python3.10 x86_64
;;
"python3.11")
build-python311-arm64
publish-python311-arm64
publish_docker_ecr $PY311_DIST_ARM64 python3.11 arm64
build-python311-x86
publish-python311-x86
publish_docker_ecr $PY311_DIST_X86_64 python3.11 x86_64
;;
"python3.12")
build-python312-arm64
publish-python312-arm64
publish_docker_ecr $PY312_DIST_ARM64 python3.12 arm64
build-python312-x86
publish-python312-x86
publish_docker_ecr $PY312_DIST_X86_64 python3.12 x86_64
;;
*)
usage
Expand Down
125 changes: 125 additions & 0 deletions ruby/publish-ecr-images.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
#!/usr/bin/env bash

set -Eeuo pipefail

# This script creates an AWS Lambda Ruby layer .zip file for each supported
# architecture. The Ruby content does not change between architectures, but
# the included Go based AWS Lambda extension does. The .zip files are written
# to dist/ and from there they are uploaded to AWS via ../libBuild.sh
# functionality.
#
# Each .zip file is structured like so for Ruby:
# ruby/gems/<RUBY MAJOR><RUBY MINOR>.0 -- AWS sets this as GEM_PATH. It's where the agent lives
# ruby/lib -- AWS sets this as RUBYLIB. It's where the wrapper script lives
# extensions/ -- Where the NR Go based extension content lives
# preview-extensions-* -- Extensions preview file

RUBY_DIR=ruby
DIST_DIR=dist
WRAPPER_FILE=newrelic_lambda_wrapper.rb
# Set this to a path to a clone of newrelic-lambda-extension to build
# an extension from scratch instead of downloading one. Set the path to ''
# to simply download a prebuilt one.
# EXTENSION_CLONE_PATH='../../newrelic-lambda-extension_fallwith'
EXTENSION_CLONE_PATH=''

source ../libBuild.sh

function usage {
echo "./publish-ecr-images.sh [ruby3.2|ruby3.3]"
}

RUBY33_DIST_ARM64=$DIST_DIR/ruby33.arm64.zip
RUBY32_DIST_ARM64=$DIST_DIR/ruby32.arm64.zip

RUBY33_DIST_X86_64=$DIST_DIR/ruby33.x86_64.zip
RUBY32_DIST_X86_64=$DIST_DIR/ruby32.x86_64.zip


function build_and_publish_ruby_for_arch {
local dist_file=$1
local ruby_version=$2
local arch=$3
echo "Building New Relic layer for ruby v$ruby_version ($arch)"


rm -rf $RUBY_DIR $dist_file
mkdir -p $DIST_DIR

bundle config set --local without development
bundle config set --local path . # Bundler will create a 'ruby' dir beneath here
bundle install

local base_dir="$RUBY_DIR/gems/$ruby_version.0"

# Bundler will have created ./ruby/<RUBY VERSION USED TO BUNDLE>/gems
# AWS wants ./ruby/gems/<RUBY VERSION FOR LAMBDA RUNTIME>
# So we need to flip the directory structure around and also use the right
# Ruby version. For building, we insist on the same major Ruby version but
# are relaxed on the minor version.
mkdir $RUBY_DIR/gems

# allow Ruby versions other than the target version to bundle
# so if Bundler used Ruby v3.3 and the target is v3.2, the '3.3.0' dir
# gets renamed to '3.2.0'
mv $RUBY_DIR/${ruby_version:0:1}* $base_dir

for sub_dir in 'bin' 'build_info' 'cache' 'doc' 'extensions' 'plugins'; do
rm -rf $base_dir/$sub_dir
done

mkdir -p $RUBY_DIR/lib
cp $WRAPPER_FILE $RUBY_DIR/lib

# if Gemfile points Bundler to GitHub for the agent, discard extraneous repo
# content and repackage the vendored gem content as if it were sourced
# from RubyGems.org
if [[ -e "$base_dir/bundler" ]]; then
local phony_version=$(date +'%s')
mkdir -p $base_dir/gems # dir will exist if non-agent, non-dev gems are in Gemfile
local nr_dir=$base_dir/gems/newrelic_rpm-$phony_version
mv $base_dir/bundler/gems/newrelic-ruby-agent* $nr_dir
rm -rf $base_dir/bundler
mkdir $base_dir/specifications
echo -e "Gem::Specification.new {|s| s.name = 'newrelic_rpm'; s.version = '$phony_version'}" > $base_dir/specifications/newrelic_rpm-$phony_version.gemspec
for sub_dir in '.git' '.github' '.gitignore' '.rubocop.yml' '.rubocop_todo.yml' '.simplecov' '.snyk' '.yardopts' 'Brewfile' 'config' 'CONTRIBUTING.md' 'docker-compose.yml' 'DOCKER.md' 'Dockerfile' 'Gemfile' 'Guardfile' 'infinite_tracing' 'init.rb' 'install.rb' 'lefthook.yml' 'newrelic.yml' 'README.md' 'test' 'THIRD_PARTY_NOTICES.md' 'Thorfile' 'recipes' '.build_ignore'; do
rm -rf "$nr_dir/$sub_dir"
done fi

if [ "$EXTENSION_CLONE_PATH" == "" ]; then
echo "Downloading prebuilt extension..."
download_extension $arch
else
echo "Building an extension from a local clone..."
here=$PWD
cd "$EXTENSION_CLONE_PATH"
make "dist-$arch"
mv "$EXTENSION_DIST_DIR" "$here/$EXTENSION_DIST_DIR"
mv "$EXTENSION_DIST_PREVIEW_FILE" "$here/$EXTENSION_DIST_PREVIEW_FILE"
cd $here
fi

zip -rq $dist_file $RUBY_DIR $EXTENSION_DIST_DIR $EXTENSION_DIST_PREVIEW_FILE
rm -rf $RUBY_DIR $EXTENSION_DIST_DIR $EXTENSION_DIST_PREVIEW_FILE
echo "Build complete: ${dist_file}"
}

set +u # permit $1 to be unbound so that '*' matches it when no args are present
case "$1" in
"ruby3.3")
build_and_publish_ruby_for_arch $RUBY33_DIST_X86_64 '3.3' 'x86_64'
publish_docker_ecr $RUBY33_DIST_X86_64 ruby3.3 x86_64
build_and_publish_ruby_for_arch $RUBY33_DIST_ARM64 '3.3' 'arm64'
publish_docker_ecr $RUBY33_DIST_ARM64 ruby3.3 arm64

;;
"ruby3.2")
build_and_publish_ruby_for_arch $RUBY32_DIST_X86_64 '3.2' 'x86_64'
publish_docker_ecr $RUBY32_DIST_X86_64 ruby3.2 x86_64
build_and_publish_ruby_for_arch $RUBY32_DIST_ARM64 '3.2' 'arm64'
publish_docker_ecr $RUBY32_DIST_ARM64 ruby3.2 arm64
;;
*)
usage
;;
esac

0 comments on commit ec5d3e0

Please sign in to comment.