From 5d5097c222d0901d776ca27090058db0f2a64d1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Portela=20Afonso?= Date: Thu, 22 Sep 2022 18:09:33 +0100 Subject: [PATCH] feat: support execution as a command (#21) --- .github/workflows/prereleased.yml | 62 +++++++++++------ .github/workflows/pull-request.yml | 7 +- .github/workflows/released.yml | 46 ++++++++----- Package.resolved | 9 +++ Package.swift | 12 +++- Sources/App/ASG/ASGClient.swift | 8 +-- Sources/App/ASG/ASGProvider.swift | 4 +- Sources/App/EKS/EKSClient.swift | 10 +-- Sources/App/EKS/EKSProvider.swift | 5 +- Sources/App/Hulk.swift | 10 ++- Sources/Command/Environment.swift | 21 ++++++ Sources/Command/run.swift | 67 +++++++++++++++++++ .../ClusterNodesTags+Codable.swift | 3 +- .../Codable => Models}/NodePool+Codable.swift | 1 - .../Codable => Models}/Tag+Codable.swift | 1 - command.Dockerfile | 35 ++++++++++ Dockerfile => lambda.Dockerfile | 2 +- 17 files changed, 241 insertions(+), 62 deletions(-) create mode 100644 Sources/Command/Environment.swift create mode 100644 Sources/Command/run.swift rename Sources/{CloudFormation/Codable => Models}/ClusterNodesTags+Codable.swift (94%) rename Sources/{CloudFormation/Codable => Models}/NodePool+Codable.swift (97%) rename Sources/{CloudFormation/Codable => Models}/Tag+Codable.swift (98%) create mode 100644 command.Dockerfile rename Dockerfile => lambda.Dockerfile (92%) diff --git a/.github/workflows/prereleased.yml b/.github/workflows/prereleased.yml index 9635f39..e3840d9 100644 --- a/.github/workflows/prereleased.yml +++ b/.github/workflows/prereleased.yml @@ -11,7 +11,8 @@ on: env: DOCKERHUB_REGISTRY: ydata - DOCKER_REPOSITORY: aws-asg-tags-lambda + DOCKER_REPOSITORY_COMMAND: aws-asg-tags-command + DOCKER_REPOSITORY_LAMBDA: aws-asg-tags-lambda @@ -51,8 +52,8 @@ jobs: run: echo ::set-output name=value::${GITHUB_REF#refs/*/} - build: - name: Build and push to Docker Hub + build-command: + name: Build and push the COMMAND version runs-on: ubuntu-20.04 needs: @@ -68,15 +69,39 @@ jobs: restore-keys: | ${{ runner.os }}-spm- - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 + - name: Login to Dockerhub Registry + uses: docker/login-action@v2 with: - role-to-assume: ${{ secrets.AWS_ECR_ROLE_ARN }} - aws-region: ${{ secrets.AWS_ECR_REGION }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to Amazon ECR - id: ecr - uses: aws-actions/amazon-ecr-login@v1 + - name: Build and push docker image + id: docker_build + uses: docker/build-push-action@v3 + env: + DOCKER_IMAGE_TAG: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.DOCKER_REPOSITORY_COMMAND }}:${{ needs.prepare.outputs.version }} + with: + file: command.Dockerfile + push: true + tags: ${{ env.DOCKER_IMAGE_TAG }} + + + build-lambda: + name: Build and push LAMBDA version + runs-on: ubuntu-20.04 + + needs: + - prepare + + steps: + - uses: actions/checkout@v3 + + - uses: actions/cache@v3 + with: + path: .build + key: ${{ runner.os }}-spm-${{ hashFiles('**/Package.resolved') }} + restore-keys: | + ${{ runner.os }}-spm- - name: Login to Dockerhub Registry uses: docker/login-action@v2 @@ -84,23 +109,16 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Build and push to AWS ECR + - name: Build and push docker image id: docker_build uses: docker/build-push-action@v3 env: - DOCKER_IMAGE_TAG: ${{ steps.ecr.outputs.registry }}/${{ env.DOCKER_REPOSITORY }}:${{ needs.prepare.outputs.version }} + DOCKER_IMAGE_TAG: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.DOCKER_REPOSITORY_LAMBDA }}:${{ needs.prepare.outputs.version }} with: + file: lambda.Dockerfile push: true tags: ${{ env.DOCKER_IMAGE_TAG }} - - name: Tag to docker hub and push - env: - SOURCE: ${{ steps.ecr.outputs.registry }}/${{ env.DOCKER_REPOSITORY }}:${{ needs.prepare.outputs.version }} - DESTINATION: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.DOCKER_REPOSITORY }}:${{ needs.prepare.outputs.version }} - run: | - docker tag $SOURCE $DESTINATION - docker push $DESTINATION - update-manifests: name: Update AWS Marketplace @@ -108,10 +126,10 @@ jobs: needs: - prepare - - build + - build-command env: - COMPONENT: ASG_TAGS_LAMBDA_VERSION + COMPONENT: ASG_TAGS_VERSION steps: - name: Checkout AWS Marketplace repo diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index d787abc..c74e9d2 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -45,5 +45,8 @@ jobs: - name: Package Resolve run: swift package resolve - - name: Build for test - run: swift build + - name: Build CloudFormation for test + run: swift build --product CloudFormation + + - name: Build Command for test + run: swift build --product Command diff --git a/.github/workflows/released.yml b/.github/workflows/released.yml index 4c8e0e2..2f68482 100644 --- a/.github/workflows/released.yml +++ b/.github/workflows/released.yml @@ -11,7 +11,8 @@ on: env: DOCKERHUB_REGISTRY: ydata - DOCKER_REPOSITORY: aws-asg-tags-lambda + DOCKER_REPOSITORY_COMMAND: aws-asg-tags-command + DOCKER_REPOSITORY_LAMBDA: aws-asg-tags-lambda @@ -65,24 +66,38 @@ jobs: run: echo ::set-output name=value::$(git tag | grep ${{ steps.short_sha.outputs.value }} | sed -r 's|([0-9].[0-9].[0-9]).*|\1|g') - docker-tag: - name: Docker Tag and Push to public and private Container Registries + docker-tag-command: + name: Docker Tag and Push COMMAND version runs-on: ubuntu-20.04 needs: - prepare steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v1 + - name: Login to Dockerhub Registry + uses: docker/login-action@v2 with: - role-to-assume: ${{ secrets.AWS_ECR_ROLE_ARN }} - aws-region: ${{ secrets.AWS_ECR_REGION }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: Login to Amazon ECR - id: ecr - uses: aws-actions/amazon-ecr-login@v1 + - name: Docker tag and push + env: + SOURCE: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.DOCKER_REPOSITORY_COMMAND }}:${{ needs.prepare.outputs.old_version }}.${{ needs.prepare.outputs.build_number }} + DESTINATION_DOCKERHUB: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.DOCKER_REPOSITORY_COMMAND }}:${{ needs.prepare.outputs.new_version }} + run: | + docker pull $SOURCE + docker tag $SOURCE $DESTINATION_DOCKERHUB + docker push $DESTINATION_DOCKERHUB + + + docker-tag-lambda: + name: Docker Tag and Push LAMBDA version + runs-on: ubuntu-20.04 + needs: + - prepare + + steps: - name: Login to Dockerhub Registry uses: docker/login-action@v2 with: @@ -91,13 +106,10 @@ jobs: - name: Docker tag and push env: - SOURCE: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.DOCKER_REPOSITORY }}:${{ needs.prepare.outputs.old_version }}.${{ needs.prepare.outputs.build_number }} - DESTINATION_ECR: ${{ steps.ecr.outputs.registry }}/${{ env.DOCKER_REPOSITORY }}:${{ needs.prepare.outputs.new_version }} - DESTINATION_DOCKERHUB: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.DOCKER_REPOSITORY }}:${{ needs.prepare.outputs.new_version }} + SOURCE: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.DOCKER_REPOSITORY_LAMBDA }}:${{ needs.prepare.outputs.old_version }}.${{ needs.prepare.outputs.build_number }} + DESTINATION_DOCKERHUB: ${{ env.DOCKERHUB_REGISTRY }}/${{ env.DOCKER_REPOSITORY_LAMBDA }}:${{ needs.prepare.outputs.new_version }} run: | docker pull $SOURCE - docker tag $SOURCE $DESTINATION_ECR - docker push $DESTINATION_ECR docker tag $SOURCE $DESTINATION_DOCKERHUB docker push $DESTINATION_DOCKERHUB @@ -108,10 +120,10 @@ jobs: needs: - prepare - - docker-tag + - docker-tag-command env: - COMPONENT: ASG_TAGS_LAMBDA_VERSION + COMPONENT: ASG_TAGS_VERSION steps: - name: Checkout AWS Marketplace repo diff --git a/Package.resolved b/Package.resolved index 9bdcc33..ce4a391 100644 --- a/Package.resolved +++ b/Package.resolved @@ -36,6 +36,15 @@ "version" : "6.0.0" } }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser", + "state" : { + "revision" : "9f39744e025c7d377987f30b03770805dcb0bcd1", + "version" : "1.1.4" + } + }, { "identity" : "swift-aws-lambda-events", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index 7451a70..b0b02ce 100644 --- a/Package.swift +++ b/Package.swift @@ -12,7 +12,8 @@ let package = Package( .package(url: "https://github.com/swift-server/swift-aws-lambda-runtime.git", branch: "main"), .package(url: "https://github.com/ydataai/swift-aws-lambda-events.git", branch: "main"), .package(url: "https://github.com/soto-project/soto.git", from: "6.0.0"), - .package(url: "https://github.com/swift-server/async-http-client.git", from: "1.11.1") + .package(url: "https://github.com/swift-server/async-http-client.git", from: "1.11.1"), + .package(url: "https://github.com/apple/swift-argument-parser", from: "1.1.4") ], targets: [ .target(name: "Models"), @@ -35,6 +36,15 @@ let package = Package( .product(name: "AWSLambdaEvents", package: "swift-aws-lambda-events") ], swiftSettings: [ .unsafeFlags(["-cross-module-optimization"], .when(configuration: .release)) ] + ), + .executableTarget( + name: "Command", + dependencies: [ + .byName(name: "App"), + .byName(name: "Models"), + .product(name: "ArgumentParser", package: "swift-argument-parser"), + ], + swiftSettings: [ .unsafeFlags(["-cross-module-optimization"], .when(configuration: .release)) ] ) ] ) diff --git a/Sources/App/ASG/ASGClient.swift b/Sources/App/ASG/ASGClient.swift index 794626b..dd52748 100644 --- a/Sources/App/ASG/ASGClient.swift +++ b/Sources/App/ASG/ASGClient.swift @@ -1,20 +1,20 @@ import Foundation import SotoAutoScaling -protocol ASGClientRepresentable { +public protocol ASGClientRepresentable { func updateTags(_ tags: [AutoScaling.Tag]) async throws } -struct ASGClient: ASGClientRepresentable { +public struct ASGClient: ASGClientRepresentable { let logger: Logger let provider: Provider - init(logger: Logger, provider: Provider) { + public init(logger: Logger, provider: Provider) { self.logger = logger self.provider = provider } - func updateTags(_ tags: [AutoScaling.Tag]) async throws { + public func updateTags(_ tags: [AutoScaling.Tag]) async throws { let updatedTags = tags.map { AutoScaling.Tag( key: $0.key, diff --git a/Sources/App/ASG/ASGProvider.swift b/Sources/App/ASG/ASGProvider.swift index a104099..9d1d2c7 100644 --- a/Sources/App/ASG/ASGProvider.swift +++ b/Sources/App/ASG/ASGProvider.swift @@ -1,13 +1,13 @@ import Foundation import SotoAutoScaling -protocol ASGProvider { +public protocol ASGProvider { func createOrUpdateTags(_ input: AutoScaling.CreateOrUpdateTagsType, logger: Logger) async throws } extension AutoScaling: ASGProvider { @inlinable - func createOrUpdateTags(_ input: CreateOrUpdateTagsType, logger: Logger) async throws { + public func createOrUpdateTags(_ input: CreateOrUpdateTagsType, logger: Logger) async throws { try await createOrUpdateTags(input, logger: logger, on: nil) } } diff --git a/Sources/App/EKS/EKSClient.swift b/Sources/App/EKS/EKSClient.swift index a47d696..881aad9 100644 --- a/Sources/App/EKS/EKSClient.swift +++ b/Sources/App/EKS/EKSClient.swift @@ -1,19 +1,19 @@ import SotoEKS -protocol EKSClientRepresentable { +public protocol EKSClientRepresentable { func describeNodeGroup(name: String, clusterName: String) async throws -> EKS.Nodegroup } -struct EKSClient: EKSClientRepresentable { +public struct EKSClient: EKSClientRepresentable { let logger: Logger let provider: Provider - init(logger: Logger, provider: Provider) { + public init(logger: Logger, provider: Provider) { self.logger = logger self.provider = provider } - func describeNodeGroup(name: String, clusterName: String) async throws -> EKS.Nodegroup { + public func describeNodeGroup(name: String, clusterName: String) async throws -> EKS.Nodegroup { let request = EKS.DescribeNodegroupRequest(clusterName: clusterName, nodegroupName: name) let response = try await provider.describeNodegroup(request, logger: logger) @@ -26,7 +26,7 @@ struct EKSClient: EKSClientRepresentable { } } -extension EKSClient { +public extension EKSClient { enum Error: Swift.Error { case cannotFindNodeGroup(_ name: String, _ clusterName: String) } diff --git a/Sources/App/EKS/EKSProvider.swift b/Sources/App/EKS/EKSProvider.swift index 3414d08..fe6d71a 100644 --- a/Sources/App/EKS/EKSProvider.swift +++ b/Sources/App/EKS/EKSProvider.swift @@ -1,14 +1,15 @@ import Foundation import SotoEKS -protocol EKSProvider { +public protocol EKSProvider { func describeNodegroup(_ input: EKS.DescribeNodegroupRequest, logger: Logger) async throws -> EKS.DescribeNodegroupResponse } extension EKS: EKSProvider { @inlinable - func describeNodegroup(_ input: DescribeNodegroupRequest, logger: Logger) async throws -> DescribeNodegroupResponse { + public func describeNodegroup(_ input: DescribeNodegroupRequest, logger: Logger) async throws + -> DescribeNodegroupResponse { try await describeNodegroup(input, logger: logger, on: nil) } } diff --git a/Sources/App/Hulk.swift b/Sources/App/Hulk.swift index 6d5c70a..30dc57b 100644 --- a/Sources/App/Hulk.swift +++ b/Sources/App/Hulk.swift @@ -3,12 +3,18 @@ import Models import SotoAutoScaling import SotoEKS -struct Hulk { +public struct Hulk { let asgClient: any ASGClientRepresentable let eksClient: any EKSClientRepresentable let logger: Logger - func smash(_ clusterInfo: ClusterNodesTags) async throws { + public init(asgClient: any ASGClientRepresentable, eksClient: any EKSClientRepresentable, logger: Logger) { + self.asgClient = asgClient + self.eksClient = eksClient + self.logger = logger + } + + public func smash(_ clusterInfo: ClusterNodesTags) async throws { logger.info("let's smash tags into ASGs with \(clusterInfo)") let asgNames = try await withThrowingTaskGroup( diff --git a/Sources/Command/Environment.swift b/Sources/Command/Environment.swift new file mode 100644 index 0000000..bf83aa2 --- /dev/null +++ b/Sources/Command/Environment.swift @@ -0,0 +1,21 @@ +import Foundation + +enum Environment { + static var decoder: JSONDecoder = JSONDecoder() + + static func get(_ key: Key) -> String? { + ProcessInfo.processInfo.environment[key.stringValue] + } + + static func get( + _ key: Key, + _ type: D.Type = D.self, + decoder: JSONDecoder = Self.decoder + ) throws -> D? { + guard let data = get(key)?.data(using: .utf8) else { + return nil + } + + return try decoder.decode(D.self, from: data) + } +} diff --git a/Sources/Command/run.swift b/Sources/Command/run.swift new file mode 100644 index 0000000..5cbd31d --- /dev/null +++ b/Sources/Command/run.swift @@ -0,0 +1,67 @@ +import App +import ArgumentParser +import Models +import NIO +import SotoCore +import SotoAutoScaling +import SotoEKS + +@main +struct Command: AsyncParsableCommand { + static var configuration: CommandConfiguration { CommandConfiguration(commandName: "aws-asg-tags") } + + + func run() async throws { + let logger = Logger(label: "ai.ydata.aws-asg-tags") + let eventLoop = MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) + + let awsClient = AWSClient( + httpClientProvider: .createNewWithEventLoopGroup(eventLoop), + logger: logger + ) + + defer { + try! awsClient.syncShutdown() // swiftlint:disable:this force_try + } + + let eks = EKS(client: awsClient) + let eksClient = EKSClient(logger: logger, provider: eks) + + let asg = AutoScaling(client: awsClient) + let asgClient = ASGClient(logger: logger, provider: asg) + + let hulk = Hulk(asgClient: asgClient, eksClient: eksClient, logger: logger) + + let clusterNodeTags = try createClusterNodeTags(logger) + + try await hulk.smash(clusterNodeTags) + } + + private func createClusterNodeTags(_ logger: Logger) throws -> ClusterNodesTags { + guard let clusterName = Environment.get(ClusterNodesTags.CodingKeys.clusterName) else { + logger.error("missing value for property \(ClusterNodesTags.CodingKeys.clusterName.description))") + throw Error.missingProperty + } + + logger.info("extracted clusterName from env: \(clusterName)") + + guard let nodePools: [NodePool] = try Environment.get(ClusterNodesTags.CodingKeys.nodePools) else { + logger.error("missing value for property \(ClusterNodesTags.CodingKeys.nodePools.description))") + throw Error.missingProperty + } + + logger.info("extracted nodePools from env: \(nodePools)") + + let commonTags: [Tag]? = try Environment.get(ClusterNodesTags.CodingKeys.commonTags) + + logger.info("commonTags extracted from env: \(commonTags ?? [])") + + return ClusterNodesTags(clusterName: clusterName, commonTags: commonTags, nodePools: nodePools) + } +} + +extension Command { + enum Error: Swift.Error { + case missingProperty + } +} diff --git a/Sources/CloudFormation/Codable/ClusterNodesTags+Codable.swift b/Sources/Models/ClusterNodesTags+Codable.swift similarity index 94% rename from Sources/CloudFormation/Codable/ClusterNodesTags+Codable.swift rename to Sources/Models/ClusterNodesTags+Codable.swift index bb220f3..8488576 100644 --- a/Sources/CloudFormation/Codable/ClusterNodesTags+Codable.swift +++ b/Sources/Models/ClusterNodesTags+Codable.swift @@ -1,8 +1,7 @@ import Foundation -import Models extension ClusterNodesTags: Codable { - enum CodingKeys: String, CodingKey { + public enum CodingKeys: String, CodingKey { case clusterName = "ClusterName" case commonTags = "CommonTags" case nodePools = "NodePools" diff --git a/Sources/CloudFormation/Codable/NodePool+Codable.swift b/Sources/Models/NodePool+Codable.swift similarity index 97% rename from Sources/CloudFormation/Codable/NodePool+Codable.swift rename to Sources/Models/NodePool+Codable.swift index 00e19f1..46a1640 100644 --- a/Sources/CloudFormation/Codable/NodePool+Codable.swift +++ b/Sources/Models/NodePool+Codable.swift @@ -1,5 +1,4 @@ import Foundation -import Models extension NodePool: Codable { enum CodingKeys: String, CodingKey { diff --git a/Sources/CloudFormation/Codable/Tag+Codable.swift b/Sources/Models/Tag+Codable.swift similarity index 98% rename from Sources/CloudFormation/Codable/Tag+Codable.swift rename to Sources/Models/Tag+Codable.swift index c5a2778..e4794d5 100644 --- a/Sources/CloudFormation/Codable/Tag+Codable.swift +++ b/Sources/Models/Tag+Codable.swift @@ -1,5 +1,4 @@ import Foundation -import Models extension Tag: Codable { enum CodingKeys: String, CodingKey { diff --git a/command.Dockerfile b/command.Dockerfile new file mode 100644 index 0000000..03f8197 --- /dev/null +++ b/command.Dockerfile @@ -0,0 +1,35 @@ +# ================================ +# Build image +# ================================ +FROM swift:5.6-focal as builder + +WORKDIR /workspace + +COPY ./Package.* ./ + +# Resolve Swift dependencies +RUN swift package resolve + +# Copy entire repo into container +# This copy the build folder to improve package resolve +COPY Sources Sources + +# Compile with optimizations +RUN swift build -c release --product Command + + +# ================================ +# Run image +# ================================ +FROM gcr.io/distroless/cc:nonroot + +LABEL org.opencontainers.image.source https://github.com/ydataai/aws-asg-tags-lambda + +COPY --from=builder /lib/x86_64-linux-gnu/libz*so* /lib/x86_64-linux-gnu/ + +# copy executables +COPY --from=builder /workspace/.build/release / +# copy Swift's dynamic libraries dependencies +COPY --from=builder /usr/lib/swift/linux/lib*so* / + +ENTRYPOINT ["/Command"] diff --git a/Dockerfile b/lambda.Dockerfile similarity index 92% rename from Dockerfile rename to lambda.Dockerfile index 870e797..e92f1d6 100644 --- a/Dockerfile +++ b/lambda.Dockerfile @@ -15,7 +15,7 @@ RUN swift package resolve COPY Sources Sources # Compile with optimizations -RUN swift build -c release +RUN swift build -c release --product CloudFormation # ================================