From 15a81a065ceb2948d0a9216b8ffb981248bdbffa Mon Sep 17 00:00:00 2001 From: Diana Perez Afanador Date: Wed, 6 Sep 2023 21:39:30 +0200 Subject: [PATCH] Update scripts for XCode Cloud --- scripts/pr-ci-matrix.rb | 112 ++++---- scripts/xcode_cloud_helper.rb | 512 ++++++++++++++++++++++++++++++++++ 2 files changed, 570 insertions(+), 54 deletions(-) create mode 100755 scripts/xcode_cloud_helper.rb diff --git a/scripts/pr-ci-matrix.rb b/scripts/pr-ci-matrix.rb index 61361dfff27..a8a0a7f1b11 100755 --- a/scripts/pr-ci-matrix.rb +++ b/scripts/pr-ci-matrix.rb @@ -1,57 +1,60 @@ #!/usr/bin/env ruby -# A script to generate the `ci_post_clone.sh` file used to run all the Xcode Cloud CI pull request job -XCODE_VERSIONS = %w(14.1 14.2 14.3.1) - -all = ->(v) { true } -latest_only = ->(v) { v == XCODE_VERSIONS.last } -oldest_and_latest = ->(v) { v == XCODE_VERSIONS.first or v == XCODE_VERSIONS.last } +# A script to generate the `ci_post_clone.sh` file used to run all the Xcode Cloud CI pull request jobs. + +module WORKFLOWS + XCODE_VERSIONS = %w(14.1 14.2 14.3.1) + + all = ->(v) { true } + latest_only = ->(v) { v == XCODE_VERSIONS.last } + oldest_and_latest = ->(v) { v == XCODE_VERSIONS.first or v == XCODE_VERSIONS.last } + + TARGETS = { + 'docs' => latest_only, + 'swiftlint' => latest_only, + + 'osx' => all, + 'osx-encryption' => latest_only, + 'osx-object-server' => oldest_and_latest, + + 'swiftpm' => oldest_and_latest, + 'swiftpm-debug' => all, + 'swiftpm-address' => latest_only, + 'swiftpm-thread' => latest_only, + 'ios-xcode-spm' => all, + + 'ios-static' => oldest_and_latest, + 'ios' => oldest_and_latest, + 'watchos' => oldest_and_latest, + 'tvos' => oldest_and_latest, + + 'osx-swift' => all, + 'ios-swift' => oldest_and_latest, + 'tvos-swift' => oldest_and_latest, + + 'osx-swift-evolution' => latest_only, + 'ios-swift-evolution' => latest_only, + 'tvos-swift-evolution' => latest_only, + + 'catalyst' => oldest_and_latest, + 'catalyst-swift' => oldest_and_latest, + + 'xcframework' => latest_only, + + 'cocoapods-osx' => all, + 'cocoapods-ios-static' => latest_only, + 'cocoapods-ios' => latest_only, + 'cocoapods-watchos' => latest_only, + 'cocoapods-tvos' => latest_only, + 'cocoapods-catalyst' => latest_only, + 'swiftui-ios' => latest_only, + 'swiftui-server-osx' => latest_only, + } +end def minimum_version(major) ->(v) { v.split('.').first.to_i >= major } end -targets = { - 'docs' => latest_only, - 'swiftlint' => latest_only, - - 'osx' => all, - 'osx-encryption' => latest_only, - 'osx-object-server' => oldest_and_latest, - - 'swiftpm' => oldest_and_latest, - 'swiftpm-debug' => all, - 'swiftpm-address' => latest_only, - 'swiftpm-thread' => latest_only, - 'ios-xcode-spm' => all, - - 'ios-static' => oldest_and_latest, - 'ios' => oldest_and_latest, - 'watchos' => oldest_and_latest, - 'tvos' => oldest_and_latest, - - 'osx-swift' => all, - 'ios-swift' => oldest_and_latest, - 'tvos-swift' => oldest_and_latest, - - 'osx-swift-evolution' => latest_only, - 'ios-swift-evolution' => latest_only, - 'tvos-swift-evolution' => latest_only, - - 'catalyst' => oldest_and_latest, - 'catalyst-swift' => oldest_and_latest, - - 'xcframework' => latest_only, - - 'cocoapods-osx' => all, - 'cocoapods-ios-static' => latest_only, - 'cocoapods-ios' => latest_only, - 'cocoapods-watchos' => latest_only, - 'cocoapods-tvos' => latest_only, - 'cocoapods-catalyst' => latest_only, - 'swiftui-ios' => latest_only, - 'swiftui-server-osx' => latest_only, -} - output_file = "#!/bin/sh" output_file << """ # This is a generated file produced by scripts/pr-ci-matrix.rb. @@ -109,8 +112,8 @@ def minimum_version(major) } : ' -xcode_version:#{XCODE_VERSIONS.map { |v| "\n - #{v}" }.join()} -target:#{targets.map { |k, v| "\n - #{k}" }.join()} +xcode_version:#{WORKFLOWS::XCODE_VERSIONS.map { |v| "\n - #{v}" }.join()} +target:#{WORKFLOWS::TARGETS.map { |k, v| "\n - #{k}" }.join()} configuration: - N/A ' @@ -123,8 +126,8 @@ def minimum_version(major) cd .. """ -targets.each_with_index { |(name, filter), index| - XCODE_VERSIONS.each { |version| +WORKFLOWS::TARGETS.each_with_index { |(name, filter), index| + WORKFLOWS::XCODE_VERSIONS.each_with_index { |version, index2| if filter.call(version) output_file << """ : ' @@ -132,15 +135,16 @@ def minimum_version(major) - target: #{name} ' """ - if index == 0 + if index == 0 && index2 == 0 output_file << "if [ \"$CI_WORKFLOW\" = \"#{name}_#{version}\" ]; then\n" + first = false else output_file << "elif [ \"$CI_WORKFLOW\" = \"#{name}_#{version}\" ]; then\n" end output_file << " export target=\"#{name}\"\n" output_file << " sh -x build.sh ci-pr | ts\n" - if index == targets.size - 1 + if index == WORKFLOWS::TARGETS.size - 1 && index2 == WORKFLOWS::XCODE_VERSIONS.size - 1 output_file << "\nelif [ \"$CI_WORKFLOW\" = \"Realm-Latest\" ] || [ \"$CI_WORKFLOW\" = \"RealmSwift-Latest\" ]; then\n" output_file << " echo \"CI workflows for testing latest XCode releases\"\n" output_file << "\nelse\n" diff --git a/scripts/xcode_cloud_helper.rb b/scripts/xcode_cloud_helper.rb new file mode 100755 index 00000000000..8d74aba91a6 --- /dev/null +++ b/scripts/xcode_cloud_helper.rb @@ -0,0 +1,512 @@ +#!/usr/bin/env ruby +require 'net/http' +require 'net/https' +require 'uri' +require 'json' +require "base64" +require "jwt" +require_relative "pr-ci-matrix" + +include WORKFLOWS + +def usage() + puts <<~END + Usage: ruby #{__FILE__} -list-workflows + Usage: ruby #{__FILE__} -list-products + Usage: ruby #{__FILE__} -list-repositories + Usage: ruby #{__FILE__} -list-mac-versions + Usage: ruby #{__FILE__} -list-xcode-versions + Usage: ruby #{__FILE__} -workflow [workflow_id] + Usage: ruby #{__FILE__} -create [workflow_id] + Usage: ruby #{__FILE__} -delete [workflow_id] + Usage: ruby #{__FILE__} -build [workflow_id] + Usage: ruby #{__FILE__} -create-new + Usage: ruby #{__FILE__} -clear-unused + + environment variables: + ISSUER_ID: Issuer Id from the App connect APIKey. + KEY_ID: Key Id from the App connect APIKey. + PK_PATH: Path to the `.p8` file containing the private key from the App connect APIKey. + PRODUCT_ID: The product Id parent for all the workflows. + REPOSITORY_ID: The repository Id pointing `realm-swift` repository. + TEAM_ID: Team id used for XCode cloud. + END + exit 1 +end + +# Apple App Connect credentials +ISSUER_ID = ENV["ISSUER_ID"] +KEY_ID = ENV["KEY_ID"] +PK_PATH = ENV["PK_PATH"] + +# XCode Cloud information +PRODUCT_ID = ENV["PRODUCT_ID"] +REPOSITORY_ID = ENV["REPOSITORY_ID"] +TEAM_ID = ENV["PK_PATH"] + +APP_STORE_URL="https://api.appstoreconnect.apple.com/v1" + +def getJwtBearer + private_key = OpenSSL::PKey.read(File.read(PK_PATH)) + token = JWT.encode( + { + iss: ISSUER_ID, + exp: Time.now.to_i + 10 * 60, + aud: "appstoreconnect-v1" + }, + private_key, + "ES256", + header_fields= + { + kid: KEY_ID + } + ) + + return token +end + +def getWorkflows + url = "#{APP_STORE_URL}/ciProducts/#{PRODUCT_ID}/workflows?limit=200" + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Get.new(uri.request_uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Accept"] = "application/json" + response = http.request(request) + + if response.code == "200" + result = JSON.parse(response.body) + + list_workflows = [] + result.collect do |doc| + doc[1].each { |workflow| + if workflow.class == Hash + name = workflow["attributes"]["name"].partition('_').first + version = workflow["attributes"]["name"].partition('_').last + list_workflows.append({ "workflow" => workflow["attributes"]["name"], "id" => workflow["id"], "target" => name, "version" => version }) + end + } + end + return list_workflows + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def getProducts + url = "#{APP_STORE_URL}/ciProducts" + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Get.new(uri.request_uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Accept"] = "application/json" + response = http.request(request) + + if response.code == "200" + result = JSON.parse(response.body) + + list_products = [] + result.collect do |doc| + doc[1].each { |prod| + if prod.class == Hash + list_products.append({ "product" => prod["attributes"]["name"], "id" => prod["id"], "type" => prod["attributes"]["productType"] }) + end + } + end + return list_products + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def getRepositories + url = "#{APP_STORE_URL}/scmRepositories" + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Get.new(uri.request_uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Accept"] = "application/json" + response = http.request(request) + + if response.code == "200" + result = JSON.parse(response.body) + + list_repositories = [] + result.collect do |doc| + doc[1].each { |repo| + if repo.class == Hash + list_repositories.append({ "name" => repo["attributes"]["repositoryName"], "id" => repo["id"], "url" => repo["attributes"]["httpCloneUrl"] }) + end + } + end + return list_repositories + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def getMacOSVersions + url = "#{APP_STORE_URL}/ciMacOsVersions" + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Get.new(uri.request_uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Accept"] = "application/json" + response = http.request(request) + + if response.code == "200" + result = JSON.parse(response.body) + + list_macosversion = [] + result.collect do |doc| + doc[1].each { |macos| + if macos.class == Hash + list_macosversion.append({ "name" => macos["attributes"]["name"], "id" => macos["id"], "version" => macos["attributes"]["version"] }) + end + } + end + return list_macosversion + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def getXCodeVersions + url = "#{APP_STORE_URL}/ciXcodeVersions" + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Get.new(uri.request_uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Accept"] = "application/json" + response = http.request(request) + + if response.code == "200" + result = JSON.parse(response.body) + + list_xcodeversion = [] + result.collect do |doc| + doc[1].each { |xcode| + if xcode.class == Hash + name = xcode["attributes"]["name"] + id = xcode["id"] + list_xcodeversion.append({ "name" => xcode["attributes"]["name"], "id" => xcode["id"], "version" => xcode["attributes"]["version"] }) + end + } + end + return list_xcodeversion + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def getWorkflowInfo(id) + url = "#{APP_STORE_URL}/ciWorkflows/#{id}" + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Get.new(uri.request_uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Accept"] = "application/json" + response = http.request(request) + + if response.code == "200" + result = JSON.parse(response.body) + name = result["data"]["attributes"]["name"] + id = result["data"]["id"] + hash = { name => id } + puts "Workflow Info:" + puts response.body + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def getUrl(url) + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Get.new(uri.request_uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Accept"] = "application/json" + response = http.request(request) + + if response.code == "200" + puts response.body + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def createWorkflow(name, xcode_version) + url = "#{APP_STORE_URL}/ciWorkflows" + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Post.new(uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Content-type"] = "application/json" + body = createWorkflowRequest(name, xcode_version) + request.body = body.to_json + response = http.request(request) + if response.code == "201" + result = JSON.parse(response.body) + id = result["data"]["id"] + puts "Worfklow created id: #{id} target: #{name} xcode version: #{xcode_version}" + puts "https://appstoreconnect.apple.com/teams/#{TEAM_ID}/frameworks/#{PRODUCT_ID}/workflows/#{id}" + return id + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def createWorkflowRequest(name, xcode_version) + build_action = + [{ + "name" => "Build - macOS", + "actionType" => "BUILD", + "destination" => "ANY_MAC", + "buildDistributionAudience" => nil, + "testConfiguration" => nil, + "scheme" => "CI", + "platform" => "MACOS", + "isRequiredToPass" => true + }] + pull_request_start_condition = + { + "source" => { "isAllMatch" => true, "patterns" => [] }, + "destination" => { "isAllMatch" => true, "patterns" => [] }, + "autoCancel" => true + } + attributes = + { + "name" => name, + "description" => name, + "isLockedForEditing" => false, + "containerFilePath" => "Realm.xcodeproj", + "isEnabled" => false, + "clean" => false, + "pullRequestStartCondition" => pull_request_start_condition, + "actions" => build_action + } + xcode_version_id = getXCodeIdForVersion(xcode_version) + mac_os_id = getMacOsLatestReleaseForXCodeVersionId(xcode_version_id) + relationships = + { + "xcodeVersion" => { "data" => { "type" => "ciXcodeVersions", "id" => "#{xcode_version_id}" }}, + "macOsVersion" => { "data" => { "type" => "ciMacOsVersions", "id" => "#{mac_os_id}" }}, + "product" => { "data" => { "type" => "ciProducts", "id" => "#{PRODUCT_ID}" }}, + "repository" => { "data" => { "type" => "scmRepositories", "id" => "#{REPOSITORY_ID}" }} + } + data = + { + "type" => "ciWorkflows", + "attributes" => attributes, + "relationships" => relationships + } + body = { "data" => data } + return body +end + +def getXCodeIdForVersion(version) + list_xcodeversion = getXCodeVersions() + list_xcodeversion.each do |xcode| + if xcode["name"].include? version + return xcode["id"] + end + end +end + +def getMacOsLatestReleaseForXCodeVersionId(xcodeVersionId) + list_macosversion = getMacOSVersionsForXCodeVersion(xcodeVersionId) + list_macosversion.each do |mac_os, id| + if mac_os.include? "Latest Release" + return id + end + end +end + +def updateWorkflow(id) + url = "#{APP_STORE_URL}/ciWorkflows/#{id}" + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Patch.new(uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Content-type"] = "application/json" + data = + { + "type" => "ciWorkflows", + "attributes" => { "isEnabled" => true }, + "id" => "#{id}" + } + body = { "data" => data } + request.body = body.to_json + response = http.request(request) + if response.code == "201" + result = JSON.parse(response.body) + id = result["data"]["id"] + puts "Worfklow updated #{id}" + return id + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def deleteWorkflow(id) + url = "#{APP_STORE_URL}/ciWorkflows/#{id}" + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Delete.new(uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Content-type"] = "application/json" + response = http.request(request) + if response.code == "204" + puts "Workflow deleted #{id}" + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def startBuild(workflow) + url = "#{APP_STORE_URL}/ciBuildRuns" + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Post.new(uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Content-type"] = "application/json" + data = + { + "type" => "ciBuildRuns", + "attributes" => {}, + "relationships" => { "workflow" => { "data" => { "type" => "ciWorkflows", "id" => "#{workflow}" }}} + } + body = { "data" => data } + request.body = body.to_json + response = http.request(request) + if response.code == "201" + result = JSON.parse(response.body) + id = result["data"]["id"] + puts "Workflow build started with id: #{id}:" + puts response.body + return id + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def getMacOSVersionsForXCodeVersion(version) + url = "#{APP_STORE_URL}/ciXcodeVersions/#{version}/macOsVersions" + uri = URI.parse(url) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + request = Net::HTTP::Get.new(uri.request_uri) + request["Authorization"] = "Bearer #{JWT_BEARER}" + request["Accept"] = "application/json" + response = http.request(request) + + if response.code == "200" + result = JSON.parse(response.body) + + list_macosversion = {} + result.collect do |doc| + doc[1].each { |macos| + if macos.class == Hash + name = macos["attributes"]["name"] + id = macos["id"] + list_macosversion.store(name, id) + end + } + end + return list_macosversion + else + puts "ERROR!!! #{response.code} #{response.body}" + end +end + +def createNewWorkflows + print "Are you sure you want to create this workflows?, this will create declared local workflows that may not currently working in other PRs [Y/N]\n" + + user_input = STDIN.gets.chomp.downcase + if user_input == "y" + workflows_to_create = [] + current_workflows = getWorkflows().map { |workflow| { "target" => workflow["target"], "version" => workflow["version"] }} + WORKFLOWS::TARGETS.each { |name, filter| + WORKFLOWS::XCODE_VERSIONS.each { |version| + if filter.call(version) + workflow = { "target" => name, "version" => version } + unless current_workflows.include? workflow + workflows_to_create.append(workflow) + end + end + } + } + + workflows_to_create.each { |workflow| + name = workflow['target'] + version = workflow['version'] + createWorkflow("#{name}_#{version}", version) + } + else + puts "No" + end +end + +def deleteUnusedWorkflows + print "Are you sure you want to clear unused workflow?, this will delete not-declared local workflows that may be currently working in other PRs [Y/N]\n" + + user_input = STDIN.gets.chomp.downcase + if user_input == "y" + local_workflows = [] + WORKFLOWS::TARGETS.each { |name, filter| + WORKFLOWS::XCODE_VERSIONS.each { |version| + if filter.call(version) + local_workflows.append("#{name}_#{version}") + end + } + } + + remote_workflows = getWorkflows + remote_workflows.each { |workflow| + unless local_workflows.include? workflow["workflow"] + puts "#{workflow["id"]} #{workflow["target"]}_#{workflow["version"]}" + deleteWorkflow(workflow["id"]) + end + } + else + puts "No" + end +end + +JWT_BEARER = getJwtBearer() +if ARGV[0] == '-list-workflows' + puts getWorkflows +elsif ARGV[0] == '-list-products' + puts getProducts +elsif ARGV[0] == '-list-repositories' + puts getRepositories +elsif ARGV[0] == '-list-mac-versions' + puts getMacOSVersions +elsif ARGV[0] == '-list-xcode-versions' + puts getXCodeVersions +elsif ARGV[0] == '-workflow' + getWorkflowInfo(ARGV[1]) +elsif ARGV[0] == '-create' + createWorkflow(ARGV[1], ARGV[2]) +elsif ARGV[0] == '-delete' + deleteWorkflow(ARGV[1]) +elsif ARGV[0] == '-build' + startBuild(ARGV[1]) +elsif ARGV[0] == '-create-new' + createNewWorkflows +elsif ARGV[0] == '-clear-unused' + deleteUnusedWorkflows +else + puts "Error, needs an argument" +end