From 34a2d14c8646f3b6fb594ee86ca1a8594269b80b Mon Sep 17 00:00:00 2001 From: Saket Chaudhary Date: Thu, 16 May 2024 13:06:07 +0530 Subject: [PATCH] update workflow --- .github/workflows/publish-node.yml | 7 ++ Dockerfile | 13 +++ libBuild.sh | 141 ++++++++++++++++++++--------- python/publish-layers.sh | 4 +- 4 files changed, 120 insertions(+), 45 deletions(-) create mode 100644 Dockerfile diff --git a/.github/workflows/publish-node.yml b/.github/workflows/publish-node.yml index 0c071a0d..390f0df0 100644 --- a/.github/workflows/publish-node.yml +++ b/.github/workflows/publish-node.yml @@ -32,6 +32,13 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: make publish-nodejs${{ matrix.node-version }}x-ci + - name: Publish ecr image for ${{ matrix.node-version }} layer + if: steps.node-check-tag.outputs.match == 'true' + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + ./publish-ecr-images.sh build-publish-nodejs${{ matrix.node-version }}x-ecr-image - name: Upload Unit Test Coverage if: steps.node-check-tag.outputs.match == 'true' uses: codecov/codecov-action@v3 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..90a5d446 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,13 @@ +FROM alpine:latest + +ARG layer_zip +ARG file_without_dist + +RUN apk update && apk add --no-cache curl unzip + +WORKDIR / + +COPY ${layer_zip} . + +RUN unzip ${file_without_dist} -d ./opt +RUN rm ${file_without_dist} \ No newline at end of file diff --git a/libBuild.sh b/libBuild.sh index ce51ffe8..684f3c27 100644 --- a/libBuild.sh +++ b/libBuild.sh @@ -4,55 +4,55 @@ set -Eeuo pipefail # Regions that support arm64 architecture REGIONS_ARM=( - af-south-1 - ap-northeast-1 - ap-northeast-2 - ap-northeast-3 - ap-south-1 - ap-southeast-1 - ap-southeast-2 - ap-southeast-3 - ca-central-1 - eu-central-1 - eu-north-1 - eu-south-1 - eu-west-1 - eu-west-2 - eu-west-3 - me-south-1 - sa-east-1 + # af-south-1 + # ap-northeast-1 + # ap-northeast-2 + # ap-northeast-3 + # ap-south-1 + # ap-southeast-1 + # ap-southeast-2 + # ap-southeast-3 + # ca-central-1 + # eu-central-1 + # eu-north-1 + # eu-south-1 + # eu-west-1 + # eu-west-2 + # eu-west-3 + # me-south-1 + # sa-east-1 us-east-1 - us-east-2 - us-west-1 + # us-east-2 + # us-west-1 us-west-2 ) REGIONS_X86=( - af-south-1 - ap-northeast-1 - ap-northeast-2 - ap-northeast-3 - ap-south-1 - ap-south-2 - ap-southeast-1 - ap-southeast-2 - ap-southeast-3 - ap-southeast-4 - ca-central-1 - eu-central-1 - eu-central-2 - eu-north-1 - eu-south-1 - eu-south-2 - eu-west-1 - eu-west-2 - eu-west-3 - me-central-1 - me-south-1 - sa-east-1 + # af-south-1 + # ap-northeast-1 + # ap-northeast-2 + # ap-northeast-3 + # ap-south-1 + # ap-south-2 + # ap-southeast-1 + # ap-southeast-2 + # ap-southeast-3 + # ap-southeast-4 + # ca-central-1 + # eu-central-1 + # eu-central-2 + # eu-north-1 + # eu-south-1 + # eu-south-2 + # eu-west-1 + # eu-west-2 + # eu-west-3 + # me-central-1 + # me-south-1 + # sa-east-1 us-east-1 - us-east-2 - us-west-1 + # us-east-2 + # us-west-1 us-west-2 ) @@ -211,7 +211,7 @@ function publish_layer { hash=$( hash_file $layer_archive | awk '{ print $1 }' ) - bucket_name="nr-layers-${region}" + bucket_name="nr-test-saket-layers-${region}" s3_key="$( s3_prefix $runtime_name )/${hash}.${arch}.zip" compat_list=( $runtime_name ) @@ -249,3 +249,56 @@ function publish_layer { --region "$region" echo "Public permissions set for ${runtime_name} layer version ${layer_version} in region ${region}" } + +function publish_docker_ecr { + layer_archive=$1 + runtime_name=$2 + arch=$3 + + if [[ ${arch} =~ 'arm64' ]]; + then arch_flag="-arm64" + else arch_flag="" + fi + + version_flag=$(echo "$runtime_name" | sed 's/[^0-9]//g') + language_flag=$(echo "$runtime_name" | sed 's/[0-9].*//') + + + # Remove 'dist/' prefix + if [[ $layer_archive == dist/* ]]; then + file_without_dist="${layer_archive#dist/}" + echo "File without 'dist/': $file_without_dist" + else + file_without_dist=$layer_archive + echo "File does not start with 'dist/': $file_without_dist" + fi + + # public ecr repository name + # maintainer can use this("q6k3q1g1") repo name for testing + repository="q6k3q1g1" + + # copy dockerfile + cp ../Dockerfile . + ls + + echo "Running : aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/${repository}" + aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/${repository} + + echo "docker build -t layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest \ + --build-arg layer_zip=${layer_archive} \ + --build-arg file_without_dist=${file_without_dist} \ + ." + + docker build -t layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest \ + --build-arg layer_zip=${layer_archive} \ + --build-arg file_without_dist=${file_without_dist} \ + . + + echo "docker tag layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}" + docker tag layer-nr-image-${language_flag}-${version_flag}${arch_flag}:latest public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag} + echo "docker push public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag}" + docker push public.ecr.aws/${repository}/newrelic-lambda-layers-${language_flag}:${version_flag}${arch_flag} + + # delete dockerfile + rm -rf Dockerfile +} diff --git a/python/publish-layers.sh b/python/publish-layers.sh index 1c21974a..4020ad46 100755 --- a/python/publish-layers.sh +++ b/python/publish-layers.sh @@ -29,7 +29,7 @@ function build-python37-x86 { echo "Building New Relic layer for python3.7 (x86_64)" rm -rf $BUILD_DIR $PY37_DIST_X86_64 mkdir -p $DIST_DIR - pip install --no-cache-dir -qU newrelic newrelic-lambda -t $BUILD_DIR/lib/python3.7/site-packages + pip3 install --no-cache-dir -qU newrelic newrelic-lambda -t $BUILD_DIR/lib/python3.7/site-packages cp newrelic_lambda_wrapper.py $BUILD_DIR/lib/python3.7/site-packages/newrelic_lambda_wrapper.py find $BUILD_DIR -name '__pycache__' -exec rm -rf {} + download_extension x86_64 @@ -288,10 +288,12 @@ function publish-python312-x86 { publish_layer $PY312_DIST_X86_64 $region python3.12 x86_64 done } + case "$1" in "python3.7") build-python37-x86 publish-python37-x86 + publish_docker_ecr $PY37_DIST_X86_64 python3.7 x86_64 ;; "python3.8") build-python38-arm64