Skip to content

Commit

Permalink
Publish docker ECR images (newrelic#229)
Browse files Browse the repository at this point in the history
* publish ecr images
  • Loading branch information
chaudharysaket authored May 22, 2024
1 parent 49ce0b8 commit dac6e90
Show file tree
Hide file tree
Showing 10 changed files with 287 additions and 1 deletion.
24 changes: 24 additions & 0 deletions .github/workflows/publish-java.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,22 @@ jobs:

steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # pin@v4
- name: Set up Java version
run: |
declare -A map_java_version
map_java_version=(
["java8al2"]="8"
["java11"]="11"
["java17"]="17"
["java21"]="21"
)
java_numeric_version=${map_java_version[${{ matrix.java-version }}]}
echo "JAVA_NUMERIC_VERSION=$java_numeric_version" >> $GITHUB_ENV
- name: Use Java ${{ env.JAVA_NUMERIC_VERSION }}
uses: actions/setup-java@v4
with:
distribution: 'corretto'
java-version: ${{ env.JAVA_NUMERIC_VERSION }}
- name: Check Tag
id: java-check-tag
run: |
Expand All @@ -26,3 +42,11 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: make publish-${{ matrix.java-version }}-ci
- name: Publish ECR image for ${{ matrix.java-version }}
if: steps.java-check-tag.outputs.match == 'true'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
cd java
./publish-layers.sh build-publish-${{ matrix.java-version }}-ecr-image
8 changes: 8 additions & 0 deletions .github/workflows/publish-node.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,14 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: make publish-nodejs${{ matrix.node-version }}x-ci
- name: Publish ECR image for nodejs${{ matrix.node-version }}
if: steps.node-check-tag.outputs.match == 'true'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
cd nodejs
./publish-layers.sh build-publish-nodejs${{ matrix.node-version }}x-ecr-image
- name: Upload Unit Test Coverage
if: steps.node-check-tag.outputs.match == 'true'
uses: codecov/codecov-action@v3
Expand Down
8 changes: 8 additions & 0 deletions .github/workflows/publish-ruby.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,11 @@ jobs:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: make publish-ruby${{ steps.ruby-version-without-dot.outputs.VERSION }}-ci
- name: Publish ECR image for ruby ${{ matrix.ruby-version }}
if: steps.ruby-check-tag.outputs.match == 'true'
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
cd ruby
./publish-ecr-images.sh ruby${{ matrix.ruby-version }}
2 changes: 1 addition & 1 deletion .github/workflows/python.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ jobs:

strategy:
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
python-version: ['3.7', '3.8', '3.9']
steps:
- uses: actions/checkout@v3

Expand Down
13 changes: 13 additions & 0 deletions Dockerfile.ecrImage
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
FROM alpine:latest

ARG layer_zip
ARG file_without_dist

RUN apk update && apk add --no-cache curl unzip

WORKDIR /

COPY ${layer_zip} .

RUN unzip ${file_without_dist} -d ./opt
RUN rm ${file_without_dist}
24 changes: 24 additions & 0 deletions java/publish-layers.sh
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,30 @@ case "$1" in
publish-java21-arm64
publish-java21-x86
;;
"build-publish-java8al2-ecr-image")
build-java8al2-arm64
publish_docker_ecr $JAVA8_DIST_ARM64 java8 arm64
build-java8al2-x86
publish_docker_ecr $JAVA8_DIST_X86_64 java8 x86_64
;;
"build-publish-java11-ecr-image")
build-java11-arm64
publish_docker_ecr $JAVA11_DIST_ARM64 java11 arm64
build-java11-x86
publish_docker_ecr $JAVA11_DIST_X86_64 java11 x86_64
;;
"build-publish-java17-ecr-image")
build-java17-arm64
publish_docker_ecr $JAVA17_DIST_ARM64 java17 arm64
build-java17-x86
publish_docker_ecr $JAVA17_DIST_X86_64 java17 x86_64
;;
"build-publish-java21-ecr-image")
build-java21-arm64
publish_docker_ecr $JAVA21_DIST_ARM64 java21 arm64
build-java21-x86
publish_docker_ecr $JAVA21_DIST_X86_64 java21 x86_64
;;
"java8al2")
$0 build-java8al2
$0 publish-java8al2
Expand Down
54 changes: 54 additions & 0 deletions libBuild.sh
Original file line number Diff line number Diff line change
Expand Up @@ -324,3 +324,57 @@ function publish_docker_ecr {


}

function publish_docker_ecr {
layer_archive=$1
runtime_name=$2
arch=$3

if [[ ${arch} =~ 'arm64' ]];
then arch_flag="-arm64"
else arch_flag=""
fi

version_flag=$(echo "$runtime_name" | sed 's/[^0-9]//g')
language_flag=$(echo "$runtime_name" | sed 's/[0-9].*//')


# Remove 'dist/' prefix
if [[ $layer_archive == dist/* ]]; then
file_without_dist="${layer_archive#dist/}"
echo "File without 'dist/': $file_without_dist"
else
file_without_dist=$layer_archive
echo "File does not start with 'dist/': $file_without_dist"
fi

# public ecr repository name
# maintainer can use this("q6k3q1g1") repo name for testing
repository="x6n7b2o2"

# copy dockerfile
cp ../Dockerfile.ecrImage .

echo "Running : aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/${repository}"
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/${repository}

echo "docker build -t layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest \
-f Dockerfile.ecrImage \
--build-arg layer_zip=${layer_archive} \
--build-arg file_without_dist=${file_without_dist} \
."

docker build -t layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest \
-f Dockerfile.ecrImage \
--build-arg layer_zip=${layer_archive} \
--build-arg file_without_dist=${file_without_dist} \
.

echo "docker tag layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}"
docker tag layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}
echo "docker push public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}"
docker push public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}

# delete dockerfile
rm -rf Dockerfile.ecrImage
}
18 changes: 18 additions & 0 deletions nodejs/publish-layers.sh
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,24 @@ case "$1" in
publish-nodejs20x-arm64
publish-nodejs20x-x86
;;
"build-publish-nodejs16x-ecr-image")
build-nodejs16x-arm64
publish_docker_ecr $NJS16X_DIST_ARM64 nodejs16.x arm64
build-nodejs16x-x86
publish_docker_ecr $NJS16X_DIST_X86_64 nodejs16.x x86_64
;;
"build-publish-nodejs18x-ecr-image")
build-nodejs18x-arm64
publish_docker_ecr $NJS18X_DIST_ARM64 nodejs18.x arm64
build-nodejs18x-x86
publish_docker_ecr $NJS18X_DIST_X86_64 nodejs18.x x86_64
;;
"build-publish-nodejs20x-ecr-image")
build-nodejs20x-arm64
publish_docker_ecr $NJS20X_DIST_ARM64 nodejs20.x arm64
build-nodejs20x-x86
publish_docker_ecr $NJS20X_DIST_X86_64 nodejs20.x x86_64
;;
"nodejs16x")
$0 build-nodejs16x
$0 publish-nodejs16x
Expand Down
12 changes: 12 additions & 0 deletions python/publish-layers.sh
Original file line number Diff line number Diff line change
Expand Up @@ -288,40 +288,52 @@ function publish-python312-x86 {
publish_layer $PY312_DIST_X86_64 $region python3.12 x86_64
done
}

case "$1" in
"python3.7")
build-python37-x86
publish-python37-x86
publish_docker_ecr $PY37_DIST_X86_64 python3.7 x86_64
;;
"python3.8")
build-python38-arm64
publish-python38-arm64
publish_docker_ecr $PY38_DIST_ARM64 python3.8 arm64
build-python38-x86
publish-python38-x86
publish_docker_ecr $PY38_DIST_X86_64 python3.8 x86_64
;;
"python3.9")
build-python39-arm64
publish-python39-arm64
publish_docker_ecr $PY39_DIST_ARM64 python3.9 arm64
build-python39-x86
publish-python39-x86
publish_docker_ecr $PY39_DIST_X86_64 python3.9 x86_64
;;
"python3.10")
build-python310-arm64
publish-python310-arm64
publish_docker_ecr $PY310_DIST_ARM64 python3.10 arm64
build-python310-x86
publish-python310-x86
publish_docker_ecr $PY310_DIST_X86_64 python3.10 x86_64
;;
"python3.11")
build-python311-arm64
publish-python311-arm64
publish_docker_ecr $PY311_DIST_ARM64 python3.11 arm64
build-python311-x86
publish-python311-x86
publish_docker_ecr $PY311_DIST_X86_64 python3.11 x86_64
;;
"python3.12")
build-python312-arm64
publish-python312-arm64
publish_docker_ecr $PY312_DIST_ARM64 python3.12 arm64
build-python312-x86
publish-python312-x86
publish_docker_ecr $PY312_DIST_X86_64 python3.12 x86_64
;;
*)
usage
Expand Down
125 changes: 125 additions & 0 deletions ruby/publish-ecr-images.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
#!/usr/bin/env bash

set -Eeuo pipefail

# This script creates an AWS Lambda Ruby layer .zip file for each supported
# architecture. The Ruby content does not change between architectures, but
# the included Go based AWS Lambda extension does. The .zip files are written
# to dist/ and from there they are uploaded to AWS via ../libBuild.sh
# functionality.
#
# Each .zip file is structured like so for Ruby:
# ruby/gems/<RUBY MAJOR><RUBY MINOR>.0 -- AWS sets this as GEM_PATH. It's where the agent lives
# ruby/lib -- AWS sets this as RUBYLIB. It's where the wrapper script lives
# extensions/ -- Where the NR Go based extension content lives
# preview-extensions-* -- Extensions preview file

RUBY_DIR=ruby
DIST_DIR=dist
WRAPPER_FILE=newrelic_lambda_wrapper.rb
# Set this to a path to a clone of newrelic-lambda-extension to build
# an extension from scratch instead of downloading one. Set the path to ''
# to simply download a prebuilt one.
# EXTENSION_CLONE_PATH='../../newrelic-lambda-extension_fallwith'
EXTENSION_CLONE_PATH=''

source ../libBuild.sh

function usage {
echo "./publish-ecr-images.sh [ruby3.2|ruby3.3]"
}

RUBY33_DIST_ARM64=$DIST_DIR/ruby33.arm64.zip
RUBY32_DIST_ARM64=$DIST_DIR/ruby32.arm64.zip

RUBY33_DIST_X86_64=$DIST_DIR/ruby33.x86_64.zip
RUBY32_DIST_X86_64=$DIST_DIR/ruby32.x86_64.zip


function build_and_publish_ruby_for_arch {
local dist_file=$1
local ruby_version=$2
local arch=$3
echo "Building New Relic layer for ruby v$ruby_version ($arch)"


rm -rf $RUBY_DIR $dist_file
mkdir -p $DIST_DIR

bundle config set --local without development
bundle config set --local path . # Bundler will create a 'ruby' dir beneath here
bundle install

local base_dir="$RUBY_DIR/gems/$ruby_version.0"

# Bundler will have created ./ruby/<RUBY VERSION USED TO BUNDLE>/gems
# AWS wants ./ruby/gems/<RUBY VERSION FOR LAMBDA RUNTIME>
# So we need to flip the directory structure around and also use the right
# Ruby version. For building, we insist on the same major Ruby version but
# are relaxed on the minor version.
mkdir $RUBY_DIR/gems

# allow Ruby versions other than the target version to bundle
# so if Bundler used Ruby v3.3 and the target is v3.2, the '3.3.0' dir
# gets renamed to '3.2.0'
mv $RUBY_DIR/${ruby_version:0:1}* $base_dir

for sub_dir in 'bin' 'build_info' 'cache' 'doc' 'extensions' 'plugins'; do
rm -rf $base_dir/$sub_dir
done

mkdir -p $RUBY_DIR/lib
cp $WRAPPER_FILE $RUBY_DIR/lib

# if Gemfile points Bundler to GitHub for the agent, discard extraneous repo
# content and repackage the vendored gem content as if it were sourced
# from RubyGems.org
if [[ -e "$base_dir/bundler" ]]; then
local phony_version=$(date +'%s')
mkdir -p $base_dir/gems # dir will exist if non-agent, non-dev gems are in Gemfile
local nr_dir=$base_dir/gems/newrelic_rpm-$phony_version
mv $base_dir/bundler/gems/newrelic-ruby-agent* $nr_dir
rm -rf $base_dir/bundler
mkdir $base_dir/specifications
echo -e "Gem::Specification.new {|s| s.name = 'newrelic_rpm'; s.version = '$phony_version'}" > $base_dir/specifications/newrelic_rpm-$phony_version.gemspec
for sub_dir in '.git' '.github' '.gitignore' '.rubocop.yml' '.rubocop_todo.yml' '.simplecov' '.snyk' '.yardopts' 'Brewfile' 'config' 'CONTRIBUTING.md' 'docker-compose.yml' 'DOCKER.md' 'Dockerfile' 'Gemfile' 'Guardfile' 'infinite_tracing' 'init.rb' 'install.rb' 'lefthook.yml' 'newrelic.yml' 'README.md' 'test' 'THIRD_PARTY_NOTICES.md' 'Thorfile' 'recipes' '.build_ignore'; do
rm -rf "$nr_dir/$sub_dir"
done fi

if [ "$EXTENSION_CLONE_PATH" == "" ]; then
echo "Downloading prebuilt extension..."
download_extension $arch
else
echo "Building an extension from a local clone..."
here=$PWD
cd "$EXTENSION_CLONE_PATH"
make "dist-$arch"
mv "$EXTENSION_DIST_DIR" "$here/$EXTENSION_DIST_DIR"
mv "$EXTENSION_DIST_PREVIEW_FILE" "$here/$EXTENSION_DIST_PREVIEW_FILE"
cd $here
fi

zip -rq $dist_file $RUBY_DIR $EXTENSION_DIST_DIR $EXTENSION_DIST_PREVIEW_FILE
rm -rf $RUBY_DIR $EXTENSION_DIST_DIR $EXTENSION_DIST_PREVIEW_FILE
echo "Build complete: ${dist_file}"
}

set +u # permit $1 to be unbound so that '*' matches it when no args are present
case "$1" in
"ruby3.3")
build_and_publish_ruby_for_arch $RUBY33_DIST_X86_64 '3.3' 'x86_64'
publish_docker_ecr $RUBY33_DIST_X86_64 ruby3.3 x86_64
build_and_publish_ruby_for_arch $RUBY33_DIST_ARM64 '3.3' 'arm64'
publish_docker_ecr $RUBY33_DIST_ARM64 ruby3.3 arm64

;;
"ruby3.2")
build_and_publish_ruby_for_arch $RUBY32_DIST_X86_64 '3.2' 'x86_64'
publish_docker_ecr $RUBY32_DIST_X86_64 ruby3.2 x86_64
build_and_publish_ruby_for_arch $RUBY32_DIST_ARM64 '3.2' 'arm64'
publish_docker_ecr $RUBY32_DIST_ARM64 ruby3.2 arm64
;;
*)
usage
;;
esac

0 comments on commit dac6e90

Please sign in to comment.