update skylight.json

.github/workflows/Gemfile

@@ -1,3 +1,5 @@
 source 'https://rubygems.org'
 
 gem 'octokit'
+gem 'rubyzip', '~> 2'
+gem 'aws-sdk-lambda'

.github/workflows/Gemfile.lock

@@ -3,18 +3,32 @@ GEM
   specs:
     addressable (2.8.7)
       public_suffix (>= 2.0.2, < 7.0)
+    aws-eventstream (1.3.0)
+    aws-partitions (1.987.0)
+    aws-sdk-core (3.209.1)
+      aws-eventstream (~> 1, >= 1.3.0)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.9)
+      jmespath (~> 1, >= 1.6.1)
+    aws-sdk-lambda (1.136.0)
+      aws-sdk-core (~> 3, >= 3.207.0)
+      aws-sigv4 (~> 1.5)
+    aws-sigv4 (1.10.0)
+      aws-eventstream (~> 1, >= 1.0.2)
     faraday (2.10.1)
       faraday-net_http (>= 2.0, < 3.2)
       logger
     faraday-net_http (3.1.1)
       net-http
+    jmespath (1.6.2)
     logger (1.6.0)
     net-http (0.4.1)
       uri
     octokit (9.1.0)
       faraday (>= 1, < 3)
       sawyer (~> 0.9)
     public_suffix (6.0.1)
+    rubyzip (2.3.2)
     sawyer (0.9.2)
       addressable (>= 2.3.5)
       faraday (>= 0.17.3, < 3)
@@ -25,7 +39,9 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  aws-sdk-lambda
   octokit
+  rubyzip (~> 2)
 
 BUNDLED WITH
    2.5.4

.github/workflows/fetch_release.rb

@@ -7,6 +7,7 @@
 require 'net/http'
 require 'digest/sha2'
 require 'octokit'
+require 'aws-sdk-lambda'
 
 root = File.expand_path('../..', __dir__)
 
@@ -18,6 +19,8 @@
 REPO = 'tildeio/skylight-otlp'
 TOKEN = ENV.fetch('GITHUB_TOKEN')
 
+ENV['AWS_REGION'] ||= 'us-east-1'
+
 def download_artifact(platform:, checksum:, version: VERSION, base_url: BASE_URL)
   filename = "skylight_otlp_#{platform}.tar.gz"
   output_filename = filename.sub('skylight_otlp', "skylight_#{version}")
@@ -44,7 +47,7 @@ def download_artifact(platform:, checksum:, version: VERSION, base_url: BASE_URL
 
   fetched_checksum = digest.hexdigest
 
-  return { checksum: checksum, path: path, platform: platform } if checksum == fetched_checksum
+  return Artifact.new(checksum: checksum, path: path, platform: platform) if checksum == fetched_checksum
 
   raise "non-matching checksum (expected = #{checksum}; actual = #{fetched_checksum} for #{platform}"
 end
@@ -55,16 +58,104 @@ def download_artifact(platform:, checksum:, version: VERSION, base_url: BASE_URL
   artifacts << download_artifact(platform: platform, checksum: checksum)
 end
 
+LAMBDA_PLATFORMS = { "x86_64-linux" => "x86_64", "aarch64-linux" => "arm64" }.freeze
+
+class Artifact
+  attr_reader :layer_version_arn
+
+  def initialize(checksum:, path:, platform:)
+    @checksum = checksum
+    @path = path
+    @platform = platform
+  end
+
+  def upload_github_asset(octokit, release)
+    puts "uploading #{@path}..."
+    asset = octokit.upload_asset(
+      release.url, 
+      @path, 
+      content_type: 'application/gzip',
+      query: { label: artifact[:platform] }
+    )
+    puts asset.url
+  end
+
+  def upload_lambda_layer(aws)
+    return unless lambda_arch = LAMBDA_PLATFORMS[@platform]
+
+    require 'zip'
+    require 'rubygems/package'
+    require 'zlib'
+
+    zipfile_path = "extension_#{lambda_arch}.zip"
+    binary_dir = "extensions/#{lambda_arch}"
+
+    FileUtils.rm_rf(zipfile_path)
+    FileUtils.rm_rf(@platform)
+    FileUtils.rm_rf(binary_dir)
+
+    FileUtils.mkdir_p(binary_dir)
+
+    # 1 - unarchive
+    Gem::Package::new("").extract_tar_gz(File.open(@path, "rb"), binary_dir)
+
+    list = Dir[File.join(binary_dir, "*")]
+
+    if list != [File.join(binary_dir, "skylight")]
+      raise "expected to find one skylight binary but found #{list.inspect}"
+    end
+
+    # 2 - make lambda layer zip file
+    Zip::File.open(zipfile_path, Zip::File::CREATE) do |zip|
+      zip.add('extensions/skylight', list[0])
+    end
+
+    # 3 - make lambda layer
+    resp = aws.publish_layer_version({
+      compatible_architectures: [lambda_arch],
+      content: {
+        zip_file: File.open(zipfile_path, 'rb')
+      }, 
+      description: "Skylight #{VERSION}", 
+      layer_name: "skylight-#{VERSION}-#{lambda_arch}".tr('.', '_'), 
+      license_info: "MIT"
+    })
+
+    @layer_arn = resp.layer_arn
+    @layer_version_arn = resp.layer_version_arn
+    @layer_version = resp.version
+
+    permission_resp = aws.add_layer_version_permission({
+      action: "lambda:GetLayerVersion", 
+      layer_name: @layer_arn, 
+      principal: "*", 
+      statement_id: "permission-#{VERSION}".tr('.', '_'), 
+      version_number: @layer_version, 
+    })
+  end
+end
+
+aws = Aws::Lambda::Client.new
+# do this first to get the ARNs
+artifacts.each do |artifact|
+  artifact.upload_lambda_layer(aws)
+end
+
 octokit = Octokit::Client.new(access_token: TOKEN)
 puts 'creating release...'
-release = octokit.create_release(REPO, VERSION,
-                                 name: "Skylight for OTLP #{VERSION}", target_commitish: 'main', draft: true, prerelease: true)
+release = octokit.create_release(
+  REPO, 
+  VERSION,
+  name: "Skylight for OTLP #{VERSION}", 
+  target_commitish: 'main', 
+  draft: true, 
+  prerelease: true,
+  # TODO: better formatting
+  body: artifacts.map(&:layer_version_arn).compact.join("\n")
+)
 
 puts "Release id #{release.id} tagged #{VERSION} (#{release.url})"
 
 artifacts.each do |artifact|
-  puts "uploading #{artifact[:path]}..."
-  asset = octokit.upload_asset(release.url, artifact[:path], content_type: 'application/gzip',
-                                                             query: { label: artifact[:platform] })
-  puts asset.url
+  artifact.upload_github_asset(octokit, release)
 end

.github/workflows/release.yml

@@ -22,4 +22,6 @@ jobs:
         id: download_artifacts
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: "bundle exec ruby .github/workflows/fetch_release.rb"

skylight.json

@@ -1,13 +1,13 @@
 {
-  "version": "6.0.0-alpha-f28dc4e",
+  "version": "6.0.0-alpha-38a1e87",
   "otlp_checksums": {
-    "x86-linux": "937e1e4add33e223f9884f54ab96ad4b4f50edc3f5a84cb9a2d7255f427aad35",
-    "x86_64-linux": "72e00dc8b736672a383cef1d4bc54c72c32e3f5a8492f9fa33c7f1cad2f9e8ef",
-    "x86_64-linux-musl": "d4802be3fd7705531a4a15bdd6f20ea3bbc4a7f3195d15155d2a7eaa4a24f70a",
-    "x86_64-darwin": "7db6764b495e7128cd811570f7ee5c4882c84cd5c8ba2b3615c6dfb90c8d575c",
-    "x86_64-freebsd": "7870c05808606fca5d0157106bc21e1113362ac1c77f1fa9ffaca54cd7b0f9da",
-    "aarch64-linux": "a662523e32a1e5396a6f77f99deab0b46a53f8ed8c926c7e3aadb8fcd2f1865c",
-    "aarch64-linux-musl": "6cc3e9d0cd56e46d693c2d73ca76f29b2d903ee3234eeb0551befe03e97444f6",
-    "aarch64-darwin": "16f6ef5c7d0f0234b9620acbe5d1c21ab20e03426e313a985442c0c0e9f6a436"
+    "x86-linux": "460512e64bae496b248edcdfa3d23b91cffb15392b637eb5039778cea5e0ebd1",
+    "x86_64-linux": "cbd6d6074f432fd9e602872f0cfa2e31d10bba83e6eba7425fb23a32f4c2d4a0",
+    "x86_64-linux-musl": "737e412186fa13a67cd2b9d1108463389adabd997b52768a48f60890b6b32c54",
+    "x86_64-darwin": "e6c6a912965bb111f172ee1a31960b4e7a13a9ac34f7bf6d695b7886b1abce44",
+    "x86_64-freebsd": "9c3577e6775d8821c31fc0a17799b67c30a2605b305214fa50c3a62916df96b4",
+    "aarch64-linux": "2ae991620fffd9c349b6143a407dc5749dae6e5e90406e87baade863eef204fe",
+    "aarch64-linux-musl": "9379ed59bd4a3ecf0855aafc74be41dc6c8f8444f32abf724d6855360652c580",
+    "aarch64-darwin": "8fbe64c34a7781a210813ed087605f2b2c7b0db5fc95b4681594f27b4636a9dc"
   }
 }