diff --git a/.github/prerequisites/README.md b/.github/prerequisites/README.md new file mode 100644 index 0000000..20f6611 --- /dev/null +++ b/.github/prerequisites/README.md @@ -0,0 +1,26 @@ +# AWS PREREQUISITES FOR THE GITHUB ACTIONS CI/CD PIPELINE + +Inspired on: + +- https://github.com/aws-actions/configure-aws-credentials/tree/main/examples + +## Part 1 (CDK Bootstrap) + +Make sure that the target account (on the needed regions) have CDK Bootstrapped already. If not, please follow these steps to have CDK init stack already in place: + +- [AWS CDK Bootstrapping Guide](https://docs.aws.amazon.com/cdk/v2/guide/bootstrapping.html) + +## Part 2 (GitHub OIDC Federation and Role) + +The CI/CD uses aws-action `configure-aws-credentials` with OIDC federation. Prior to using this project, we need to deploy the [github-actions-oidc-federation-and-role](github-actions-oidc-federation-and-role.yml) CloudFormation template in the AWS account(s) that we need to deploy the solution. Specify the GitHub Organization Name, Repository Name, and the specific branch you want to deploy on. + +> Note: In this case, "GitHub Organization Name" is "san99tiago" (my GitHub account). + +To use the example you will need to set the following GitHub Action Secrets: + +| Secret Key | Used With | Description | +| -------------------- | ------------------------- | ------------------------------- | +| DEV_AWS_ACCOUNT_ID | configure-aws-credentials | The AWS account ID (DEV) | +| PROD_AWS_ACCOUNT_ID | configure-aws-credentials | The AWS account ID (PROD) | +| DEV_AWS_DEPLOY_ROLE | configure-aws-credentials | The name of the IAM role (DEV) | +| PROD_AWS_DEPLOY_ROLE | configure-aws-credentials | The name of the IAM role (PROD) | diff --git a/.github/prerequisites/github-actions-oidc-federation-and-role.yml b/.github/prerequisites/github-actions-oidc-federation-and-role.yml new file mode 100644 index 0000000..cc8cdc9 --- /dev/null +++ b/.github/prerequisites/github-actions-oidc-federation-and-role.yml @@ -0,0 +1,92 @@ +--- +AWSTemplateFormatVersion: "2010-09-09" +Description: Github Actions configuration - OIDC IAM IdP and associated role CI/CD + +Parameters: + + GitHubOrganization: + Type: String + Description: This is the root organization or personal account where repos are stored (Case Sensitive) + + RepositoryName: + Type: String + Description: The repo(s) these roles will have access to. (Use * for all org or personal repos) + Default: "*" + + BranchName: + Type: String + Description: Name of the git branch to to trust. (Use * for all branches) + Default: "*" + + RoleName: + Type: String + Description: Name the Role + + UseExistingProvider: + Type: String + Description: "Only one GitHub Provider can exists. Choose yes if one is already present in account" + Default: "no" + AllowedValues: + - "yes" + - "no" + +Conditions: + + CreateProvider: !Equals ["no", !Ref UseExistingProvider] + +Resources: + + IdpGitHubOidc: + Type: AWS::IAM::OIDCProvider + Condition: CreateProvider + Properties: + Url: https://token.actions.githubusercontent.com + ClientIdList: + - sts.amazonaws.com + - !Sub https://github.com/${GitHubOrganization}/${RepositoryName} + ThumbprintList: + - 6938fd4d98bab03faadb97b34396831e3780aea1 + Tags: + - Key: Name + Value: !Sub ${RoleName}-OIDC-Provider + + RoleGithubActions: + Type: AWS::IAM::Role + Properties: + RoleName: !Ref RoleName + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Action: sts:AssumeRoleWithWebIdentity + Principal: + Federated: !If + - CreateProvider + - !Ref IdpGitHubOidc + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:oidc-provider/token.actions.githubusercontent.com + Condition: + StringLike: + token.actions.githubusercontent.com:sub: !Sub repo:${GitHubOrganization}/${RepositoryName}:ref:refs/heads/${BranchName} + + RoleGithubActionsPolicies: + Type: "AWS::IAM::Policy" + Properties: + PolicyName: !Sub ${RoleName}-Policy + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: "sts:AssumeRole" + Resource: "arn:aws:iam::*:role/cdk-*" + Roles: + - !Ref RoleGithubActions + +Outputs: + + IdpGitHubOidc: + Condition: CreateProvider + Description: "ARN of Github OIDC Provider" + Value: !GetAtt IdpGitHubOidc.Arn + + RoleGithubActionsARN: + Description: "CICD Role for GitHub Actions" + Value: !GetAtt RoleGithubActions.Arn \ No newline at end of file diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..ff8986a --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,237 @@ +name: deploy + +on: + push: + branches: ["main", "develop"] + +env: + AWS_DEFAULT_REGION: us-east-1 + AWS_DEFAULT_OUTPUT: json + +jobs: + code-quality: + name: Check Coding Standards + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Runner Details + run: | + echo "Job triggered by ${{ github.event_name }} event." + echo "Job running on a ${{ runner.os }} server hosted by GitHub." + echo "Pipeline's branch name is ${{ github.ref }} and repository is ${{ github.repository }}." + + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Install Poetry dependencies + run: poetry install --no-interaction + + - name: Check code formatting + run: poetry run poe black-check + + unit-tests: + name: Run Unit Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Install Poetry dependencies + run: poetry install --no-interaction + + - name: Run Unit Tests + run: poetry run poe test-unit + + - name: Archive code coverage results + uses: actions/upload-artifact@v3 + with: + name: code-coverage-report + path: htmlcov + + cdk-synth-diff: + name: CDK Synth & Diff + runs-on: ubuntu-latest + needs: ["code-quality", "unit-tests"] + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Install Poetry dependencies + run: poetry install --no-interaction + + - name: Set up NodeJs + uses: actions/setup-node@v3 + with: + node-version: "20" + + - name: Install CDK + run: npm install -g aws-cdk + + # Same task with different secrets depending on the branch ref (dev vs prod deployments) + # Note: there might be better alternatives, but this is a way to deploy to both envs + - name: Configure AWS Credentials (DEV) + if: github.ref != 'refs/heads/main' + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.AWS_DEFAULT_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.DEV_AWS_ACCOUNT_ID }}:role/${{ secrets.DEV_AWS_DEPLOY_ROLE }} + role-session-name: GitHubActionsCICD + + - name: Configure AWS Credentials (PROD) + if: github.ref == 'refs/heads/main' + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.AWS_DEFAULT_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.PROD_AWS_ACCOUNT_ID }}:role/${{ secrets.PROD_AWS_DEPLOY_ROLE }} + role-session-name: GitHubActionsCICD + + # Not 100% optimal, but does the work for setting deployment environments from branch + - name: Setup Deployment Environment from Branch + run: | + # To update the deployment environment based on the git branch + if [[ $GITHUB_REF == 'refs/heads/main' ]]; then + echo "DEPLOYMENT_ENVIRONMENT=prod" >> "$GITHUB_ENV" + echo "Deployment environment is PROD" + else + echo "DEPLOYMENT_ENVIRONMENT=dev" >> "$GITHUB_ENV" + echo "Deployment environment is DEV" + fi + + - name: CDK Synth + run: cdk synth + + - name: CDK Diff + run: cdk diff + + - name: Archive CDK Synth results (no assets) + uses: actions/upload-artifact@v3 + with: + name: cdk-synth-folder + path: | + ./cdk.out + !./cdk.out/asset.* + retention-days: 1 + + iac-checkov: + name: IaC Checkov Validations + runs-on: ubuntu-latest + needs: cdk-synth-diff + steps: + - uses: actions/checkout@v3 + + - name: Dowload CDK Synth results + uses: actions/download-artifact@v3 + with: + name: cdk-synth-folder + path: ./cdk-synth-output-folder + + - name: Display files in the output folder + run: tree + working-directory: ./cdk-synth-output-folder + + - name: Run Checkov action + id: checkov + uses: bridgecrewio/checkov-action@v12 + with: + directory: cdk-synth-output-folder/ + framework: cloudformation + soft_fail: true # optional: do not return an error code if there are failed checks + skip_check: CKV_AWS_2 # optional: skip a specific check_id. can be comma separated list + quiet: true # optional: display only failed checks + + cdk-deploy: + name: CDK Deploy + runs-on: ubuntu-latest + needs: "cdk-synth-diff" + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: 3.11 + + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + - name: Install Poetry dependencies + run: poetry install --no-interaction + + - name: Set up NodeJs + uses: actions/setup-node@v3 + with: + node-version: "20" + + - name: Install CDK + run: npm install -g aws-cdk + + # Same task with different secrets depending on the branch ref (dev vs prod deployments) + # Note: there might be better alternatives, but this is a way to deploy to both envs + - name: Configure AWS Credentials (DEV) + if: github.ref != 'refs/heads/main' + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.AWS_DEFAULT_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.DEV_AWS_ACCOUNT_ID }}:role/${{ secrets.DEV_AWS_DEPLOY_ROLE }} + role-session-name: GitHubActionsCICD + + - name: Configure AWS Credentials (PROD) + if: github.ref == 'refs/heads/main' + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ env.AWS_DEFAULT_REGION }} + role-to-assume: arn:aws:iam::${{ secrets.PROD_AWS_ACCOUNT_ID }}:role/${{ secrets.PROD_AWS_DEPLOY_ROLE }} + role-session-name: GitHubActionsCICD + + # Not 100% optimal, but does the work for setting deployment environments from branch + - name: Setup Deployment Environment from Branch + run: | + # To update the deployment environment based on the git branch + if [[ $GITHUB_REF == 'refs/heads/main' ]]; then + echo "DEPLOYMENT_ENVIRONMENT=prod" >> "$GITHUB_ENV" + echo "Deployment environment is PROD" + else + echo "DEPLOYMENT_ENVIRONMENT=dev" >> "$GITHUB_ENV" + echo "Deployment environment is DEV" + fi + + - name: CDK Deploy + run: cdk deploy --require-approval never diff --git a/README.md b/README.md index 7a89665..ab366d5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # :scroll: AWS-WHATSAPP-CHATBOT :scroll: +![Latest CI/CD Action workflow](https://github.com/san99tiago/aws-whatsapp-chatbot/actions/workflows/deploy.yml/badge.svg?branch=main) + WhatsApp Chatbot on AWS that enables the creation of a customized multi-modal "personal assistant" for general purposes (customized with my own data and requirements). ## Architecture :memo: diff --git a/pyproject.toml b/pyproject.toml index 8ef4cef..d5e1cf0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,11 @@ build-backend = "poetry.core.masonry.api" [tool.poe.tasks] black-format = "black ." black-check = "black . --check --diff -v" -test-unit = ["_test_unit", "_coverage_html"] +test-unit = ["_test_unit", "_coverage_report", "_coverage_html"] synth = "cdk synth" deploy-backend-dev = "cdk deploy investments-portfolio-tracker-backend-dev --require-approval never" _test_unit = "coverage run -m pytest tests/unit" +_coverage_report = "coverage report" _coverage_html = "coverage html" [tool.coverage.run] diff --git a/tests/unit/cdk/test_cdk_chatbot_api_stack.py b/tests/unit/cdk/test_cdk_chatbot_api_stack.py index 591142c..c20355b 100644 --- a/tests/unit/cdk/test_cdk_chatbot_api_stack.py +++ b/tests/unit/cdk/test_cdk_chatbot_api_stack.py @@ -19,6 +19,7 @@ "table_name": "aws-whatsapp-poc-test", "api_gw_name": "wpp-test", "secret_name": "test-secret", + "meta_endpoint": "https://fake-endpoint.com", }, ) template: assertions.Template = assertions.Template.from_stack(stack)