feat(collector): introduce new workflow to publish collector Lambda layer

.github/workflows/layer-publish.yml

@@ -32,7 +32,14 @@ on:
         description: 'Publish to which AWS region?'
         required: true
         type: string
-
+      role-arn:
+        description: '(optional) AWS IAM Role ARN to be assumed for publishing layer. If no input is given, defaults to `PROD_LAMBDA_ROLE_ARN` secret.'
+        required: false
+        type: string
+      layer-version:
+        description: '(optional) Layer version to be used in the layer name. If no input is given, its value is tried to be extracted from the `GITHUB_REF_NAME` variable'
+        required: false
+        type: string
 
 permissions:
   id-token: write
@@ -58,18 +65,24 @@ jobs:
           echo "ARCH=$ARCH" >> $GITHUB_ENV
 
           if [[ -n "${{ inputs.runtimes }}" ]]; then
-            RUNTIMES="--compatible-runtimes ${{ inputs.runtimes }}"
+            COMPATIBLE_RUNTIMES="--compatible-runtimes ${{ inputs.runtimes }}"
           fi
-          echo "RUNTIMES=$RUNTIMES" >> $GITHUB_ENV
+          echo "COMPATIBLE_RUNTIMES=$COMPATIBLE_RUNTIMES" >> $GITHUB_ENV
 
           if [[ "${{ inputs.release-group }}" != "prod" ]]; then
             LAYER_NAME=$LAYER_NAME-${{ inputs.release-group }}
           fi
 
-          LAYER_VERSION=$(echo "$GITHUB_REF_NAME" | sed -r 's/.*\/[^0-9\.]*//g')
-          LAYER_VERSION_CLEANED=$(echo "$LAYER_VERSION" | sed -r 's/\./_/g')
+          if [[ -n "${{ inputs.layer-version }}" ]]; then
+            LAYER_VERSION="${{ inputs.layer-version }}"
+          else
+            LAYER_VERSION=$(echo "$GITHUB_REF_NAME" | sed -r 's/.*\/[^0-9\.]*//g')
+          fi
+          LAYER_VERSION_CLEANED=$(echo "$LAYER_VERSION" | sed -r 's/\./_/g')          
 
-          LAYER_NAME=$LAYER_NAME-$LAYER_VERSION_CLEANED
+          if [[ -n "$LAYER_VERSION_CLEANED" ]]; then
+            LAYER_NAME=$LAYER_NAME-$LAYER_VERSION_CLEANED
+          fi
           echo "LAYER_NAME=$LAYER_NAME" >> $GITHUB_ENV
 
           echo GITHUB_ENV:
@@ -82,7 +95,7 @@ jobs:
 
       - uses: aws-actions/configure-aws-credentials@v4
         with:
-          role-to-assume: ${{ secrets.PROD_LAMBDA_ROLE_ARN }}
+          role-to-assume: ${{ inputs.role-arn || secrets.OTEL_LAMBDA_LAYER_PUBLISH_ROLE_ARN || secrets.PROD_LAMBDA_ROLE_ARN }}
           role-duration-seconds: 1200
           aws-region: ${{ inputs.aws_region }}
           mask-aws-account-id: false
@@ -93,7 +106,8 @@ jobs:
             aws lambda publish-layer-version \
               --layer-name $LAYER_NAME \
               --license-info "Apache 2.0" \
-              --compatible-architectures $ARCH $RUNTIMES \
+              --compatible-architectures $ARCH \
+              $COMPATIBLE_RUNTIMES \
               --zip-file fileb://${{ inputs.artifact-name }} \
               --query 'LayerVersionArn' \
               --output text

.github/workflows/publish-layer-collector.yml

@@ -0,0 +1,131 @@
+name: "Publish Collector Lambda layer"
+
+on:
+  workflow_dispatch:
+    inputs:
+      architecture:
+        description: 'Architecture of the layer to be published'
+        required: true
+        type: choice
+        options:
+          - all
+          - amd64
+          - arm64
+        default: all
+      aws-region:
+        description: 'AWS Region(s) where layer will be published'
+        required: true
+        type: choice
+        options:
+          - all
+          - ap-northeast-1
+          - ap-northeast-2
+          - ap-south-1
+          - ap-southeast-1
+          - ap-southeast-2
+          - ca-central-1
+          - eu-central-1
+          - eu-north-1
+          - eu-west-1
+          - eu-west-2
+          - eu-west-3
+          - sa-east-1
+          - us-east-1
+          - us-east-2
+          - us-west-1
+          - us-west-2
+        default: all
+      role-arn:
+        description: 'AWS IAM Role ARN to be assumed for publishing layer'
+        required: false
+        type: string
+      layer-version:
+        description: 'Layer version to be appended into the layer name'
+        required: false
+        type: string
+      build-tags:
+        description: 'Build tags to customize collector build'
+        required: false
+        type: string
+
+jobs:
+  prepare-build-jobs:
+    runs-on: ubuntu-latest
+    outputs:
+      build_jobs: ${{ steps.prepare-build-jobs.outputs.build_jobs }}
+    steps:
+      - id: prepare-build-jobs
+        name: Prepare Build Jobs
+        run: |
+          architectures=''
+          if [ ${{ github.event.inputs.architecture }} == 'all' ]; then
+            architectures='["amd64", "arm64"]'
+          else
+            architectures='["${{ github.event.inputs.architecture }}"]'
+          fi
+          echo "build_jobs={"architecture": ${architectures}}" | tr -d '[:space:]' >> $GITHUB_OUTPUT
+  build-layer:
+    needs: prepare-build-jobs
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{ fromJSON(needs.prepare-build-jobs.outputs.build_jobs) }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '~1.21.9'
+      - name: Build Collector
+        run: |
+          if [[ -n "${{ inputs.build-tags }}" ]]; then
+            BUILDTAGS="${{ inputs.build-tags }}"
+            if [[ "$BUILDTAGS" != "lambdacomponents.custom"* ]]; then
+              BUILDTAGS="lambdacomponents.custom,$BUILDTAGS"
+            fi
+          fi
+          echo "Build tags: $BUILDTAGS"
+          make -C collector package GOARCH=${{ matrix.architecture }} BUILDTAGS=$BUILDTAGS
+      - name: Upload Collector Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: opentelemetry-collector-layer-${{ matrix.architecture }}.zip
+          path: ${{ github.workspace }}/collector/build/opentelemetry-collector-layer-${{ matrix.architecture }}.zip
+  prepare-release-jobs:
+    needs: build-layer
+    runs-on: ubuntu-latest
+    outputs:
+      release_jobs: ${{ steps.prepare-release-jobs.outputs.release_jobs }}
+    steps:
+      - id: prepare-release-jobs
+        name: Prepare Release Jobs
+        run: |
+          architectures=''
+          if [ ${{ github.event.inputs.architecture }} == 'all' ]; then
+            architectures='["amd64", "arm64"]'
+          else
+            architectures='["${{ github.event.inputs.architecture }}"]'
+          fi
+          aws_regions=''
+          if [ ${{ github.event.inputs.aws-region }} == 'all' ]; then
+            aws_regions='["ap-northeast-1", "ap-northeast-2", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "eu-west-3", "sa-east-1", "us-east-1", "us-east-2", "us-west-1", "us-west-2"]'
+          else
+            aws_regions='["${{ github.event.inputs.aws-region }}"]'
+          fi
+          echo "release_jobs={"architecture": ${architectures}, "aws_region": ${aws_regions}}" | tr -d '[:space:]' >> $GITHUB_OUTPUT
+  release-layer:
+    uses: ./.github/workflows/layer-publish.yml
+    needs: prepare-release-jobs
+    strategy:
+      matrix: ${{ fromJSON(needs.prepare-release-jobs.outputs.release_jobs) }}
+    with:
+      artifact-name: opentelemetry-collector-layer-${{ matrix.architecture }}.zip
+      layer-name: opentelemetry-collector
+      architecture: ${{ matrix.architecture }}
+      runtimes: "nodejs16.x nodejs18.x nodejs20.x nodejs22.x java11 java17 java21 python3.8 python3.9 python3.10 python3.11 python3.12"
+      release-group: prod
+      aws_region: ${{ matrix.aws_region }}
+      role-arn: ${{ github.event.inputs.role-arn }}
+      component-version: 'NA'
+      layer-version: ${{ github.event.inputs.layer-version }}
+    secrets: inherit

collector/README.md

@@ -52,6 +52,40 @@ For example, if you want to add the extension `foo`, the file providing this ext
 
 You can provide your addition as a pull-request to this repository. Before doing so, please also read through the details of [Contributing](#contributing) to this project.
 
+## Build and publish your own OpenTelemetry Collector Lambda layer
+
+To build and publish collector Lambda layer from your own fork into your own AWS account, 
+you can use the `Publish Collector Lambda Layer` workflow which can only be triggered manually.
+
+
+To do that, first you need to 
+- Create Github's OIDC provider in your (or target) AWS account (for more details, you can check [here](https://github.com/aws-actions/configure-aws-credentials?oidc))
+- Create an AWS IAM Role in the AWS account to be assumed by the `Publish Collector Lambda Layer` workflow from your forked OpenTelemetry Lambda repository.
+
+To setup those, you can use (copy or load) the AWS CloudFormation template [here](../utils/aws-cloudformation/aws-cf-stack-for-layer-publish.yml).
+Once AWS CloudFormation stack is created from the given template, 
+ARN of the created AWS IAM Role to be assumed will be shown as `RoleARN` in the output of the stack, so note it to be used later.
+
+After that, you can run the `Publish Collector Lambda Layer` workflow to build the Lambda collector and publish it to the target AWS account as Lambda layer: 
+- Specify the architecture of the collector Lambda layer to be published via the `Architecture of the layer to be published` input. 
+  Available options are `all`, `amd64` and `arm64`.
+  The default value is `all` which builds and publishes layer for both of the `amd64` and `arm64` architectures.
+- Specify the AWS region(s) where the collector Lambda layer will be published to via the `AWS Region(s) where layer will be published` input.
+  Available options are `all`, `ap-northeast-1`, `ap-northeast-2`, `ap-south-1`, `ap-southeast-1`, `ap-southeast-2`, `ca-central-1`, `eu-central-1`, `eu-north-1`, `eu-west-1`, `eu-west-2`, `eu-west-3`, `sa-east-1`, `us-east-1`, `us-east-2`, `us-west-1`, `us-west-2`.
+  The default value is `all` which publishes layer to all the defined AWS regions mentioned above.
+- Specify the AWS IAM Role ARN to be assumed for publishing layer via the `AWS IAM Role ARN to be assumed for publishing layer` input.
+  This is the ARN of the AWS IAM Role you have taken from the `RoleARN` output variable of the created AWS CloudFormation stack above.
+  This input is **optional** and if not specified, AWS IAM Role ARN to be assumed is tried to be resolved from `OTEL_LAMBDA_LAYER_PUBLISH_ROLE_ARN` secret.
+  If it is still not able to resolved (neither this input is specified, nor `OTEL_LAMBDA_LAYER_PUBLISH_ROLE_ARN` secret is defined), 
+  layer publish job will fail due to missing AWS credentials.
+- Specify the layer version to be appended into layer name via the `Layer version to be appended into the layer name` input 
+  to be used in the following format: `opentelemetry-lambda-collector-${architecture}-${layer-version}`.
+  This input is **optional** and if not specified, layer name is generated in the `opentelemetry-lambda-collector-${architecture}` format without layer version postfix.
+- Specify the build tags to build the collector with a customized set of connectors/exporters/receivers/processors 
+  via the `Build tags to customize collector build` input.
+  This input is **optional** and if not specified, collector is built with the default set of connectors/exporters/receivers/processors.
+  Check the [Build Tags](#build-tags) section for the details.
+
 ## Installing
 To install the OpenTelemetry Collector Lambda layer to an existing Lambda function using the `aws` CLI:
 

utils/aws-cloudformation/aws-cf-stack-for-layer-publish.yml

@@ -0,0 +1,53 @@
+Parameters:
+  GitHubOrgName:
+    Description: Name of the GitHub organization/user
+    Type: String
+  RepositoryName:
+    Description: Name of the GitHub repository
+    Type: String
+    Default: "opentelemetry-lambda"
+
+Resources:
+  Role:
+    Type: AWS::IAM::Role
+    Properties:
+      RoleName: "github-otel-lambda-layer-publish-role"
+      AssumeRolePolicyDocument:
+        Statement:
+          - Effect: Allow
+            Action: sts:AssumeRoleWithWebIdentity
+            Principal:
+              Federated: !Ref GithubOIDC
+            Condition:
+              StringEquals:
+                token.actions.githubusercontent.com:aud: "sts.amazonaws.com"
+              StringLike:
+                token.actions.githubusercontent.com:sub: !Sub "repo:${GitHubOrgName}/${RepositoryName}:*"
+      Policies:
+        - PolicyName: "github-otel-lambda-layer-publish-policy"
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: Allow
+                Action:
+                  - "lambda:GetLayer*"
+                  - "lambda:ListLayer*"
+                  - "lambda:AddLayer*"
+                  - "lambda:PublishLayer*"
+                Resource:
+                  - !Sub "arn:aws:lambda:*:${AWS::AccountId}:layer:opentelemetry-*"
+                  - !Sub "arn:aws:lambda:*:${AWS::AccountId}:layer:opentelemetry-*:*"
+
+  GithubOIDC:
+    Type: AWS::IAM::OIDCProvider
+    Properties:
+      Url: "https://token.actions.githubusercontent.com"
+      ClientIdList: 
+        - "sts.amazonaws.com"
+      ThumbprintList:
+        - "ffffffffffffffffffffffffffffffffffffffff"
+
+Outputs:
+  RoleARN:
+    Description: "ARN of the AWS IAM role to be assumed by Github for the OpenTelemetry Layer publishing"
+    Value: !GetAtt Role.Arn