From 955a0c92d93f5695a2f77208f16ee2ed420b89ff Mon Sep 17 00:00:00 2001 From: vggonzal <9Tcostoamm> Date: Wed, 2 Aug 2023 21:30:46 -0700 Subject: [PATCH] update terraform to provision endpoints --- .github/workflows/build.yml | 2 +- docker/Dockerfile | 7 +++ docker/README.md | 48 +++++++++++++++++++ docker/build-docker.sh | 60 ++++++++++++++++++++++++ docker/push-docker-artifactory.sh | 66 +++++++++++++++++++++++++++ docker/push-docker-ecr.sh | 76 +++++++++++++++++++++++++++++++ 6 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 docker/Dockerfile create mode 100644 docker/README.md create mode 100755 docker/build-docker.sh create mode 100755 docker/push-docker-artifactory.sh create mode 100755 docker/push-docker-ecr.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7d63bcc..4c542d8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -103,7 +103,7 @@ jobs: # Setup Terraform to Deploy - - + - name: Configure AWS Credentials as Environment Variables run: echo "AWS_ACCESS_KEY_ID=${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}" >> $GITHUB_ENV | echo "AWS_SECRET_ACCESS_KEY=${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}" >> $GITHUB_ENV diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..d76e57d --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,7 @@ +FROM public.ecr.aws/lambda/python:3.8 + +COPY $SOURCE . +RUN pip3 install -t /var/task --force ./$SOURCE + +# Run the lambda +CMD ["/var/task/podaac/controllers/fts_controller.lambda_handler"] \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..c70bb67 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,48 @@ +# FTS API Docker Image + +This directory contains the `Dockerfile` used to build the Docker image capable of running FTS API as a lambda. + +It includes a number of helper scripts to be run by the CI/CD pipeline but can also be run locally to build the image. + +## Building + +Building the FTS API docker image depends on a tar file version of the project. This can be built using `poetry build` or by downloading a previously built version of the project as a tar. + +### Building from tar + +`build-docker.sh` script can be used to build the docker image from the +local tar file. There are two required arguments that must be set: + +1. service-name: The name of the service being built (from pyproject.toml) +2. service-version: The version of the service being built (also from pyproject.toml) + +The docker tag of the built image will be returned from the script. + +Example: + +```shell script +./docker/build-docker.sh -n podaac-fts -v 1.0.0-alpha.3 +``` + +## Running + +The Docker image can be run directly using the `docker run` command. + +See [Testing Lambda container images locally](https://docs.aws.amazon.com/lambda/latest/dg/images-test.html) for details. + +## Pushing to ECR + +The `push-docker-ecr.sh` script can be used to push a docker image to AWS ECR. There are two required arguments: + +1. tf-venue: The target venue for uploading (sit, uat, or ops). +2. docker-tag: The docker tage of the image being pushed + +The easiest way to use the `push-docker-ecr.sh` script is to first call `build-docker.sh` and save the output to the +`docker_tag` environment variable. Then call `push-docker-ecr.sh`. + +Example: + +```shell script +export docker_tag=$(./docker/build-docker.sh -n podaac-fts -v 1.0.0-alpha.3) +./docker/push-docker-ecr.sh -v sit -t $docker_tag +``` \ No newline at end of file diff --git a/docker/build-docker.sh b/docker/build-docker.sh new file mode 100755 index 0000000..7f22d13 --- /dev/null +++ b/docker/build-docker.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# This script is intended to be run by the CI/CD pipeline to build a specific version of the FTS API. + +set -Eeo pipefail + +POSITIONAL=() +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + -n|--service-name) + service_name="$2" + shift # past argument + shift # past value + ;; + -v|--service-version) + service_version="$2" + shift # past argument + shift # past value + ;; + *) # unknown option + POSITIONAL+=("$1") # save it in an array for later + shift # past argument + ;; +esac +done +set -- "${POSITIONAL[@]}" # restore positional parameters + +USAGE="USAGE: build-docker.sh -n|--service-name service_name -v|--service-version service_version" + +# shellcheck disable=SC2154 +if [[ -z "${service_name}" ]]; then + echo "service_name required. Name of the service as found in pyproject.toml (e.g. podaac-staging)" >&2 + echo "$USAGE" >&2 + exit 1 +fi + +# shellcheck disable=SC2154 +if [[ -z "${service_version}" ]]; then + echo "service_version required. Version of software to install (e.g. 0.1.0-a1+12353)." >&2 + echo "$USAGE" >&2 + exit 1 +fi + +set -u + +SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" +PROJECT_DIR="$(dirname "${SCRIPTPATH}")" + +repositoryName=podaac/podaac-cloud/${service_name} + +# Docker tags can't include '+' https://github.com/docker/distribution/issues/1201 +dockerTagVersion=$(echo "${service_version}" | tr "+" _) + +tar_filename="${service_name}-${service_version}.tar.gz" +docker build -t "${repositoryName}":"${dockerTagVersion}" --build-arg SOURCE="dist/${tar_filename}" -f "$SCRIPTPATH"/Dockerfile "$PROJECT_DIR" 1>&2 + +echo "${repositoryName}":"${dockerTagVersion}" \ No newline at end of file diff --git a/docker/push-docker-artifactory.sh b/docker/push-docker-artifactory.sh new file mode 100755 index 0000000..c84d2cd --- /dev/null +++ b/docker/push-docker-artifactory.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +# This script is intended to be run by the CI/CD pipeline to push a docker tag previously built by build-docker.sh + +set -Eeo pipefail + +POSITIONAL=() +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + -t|--docker-tag) + docker_tag="$2" + shift # past argument + shift # past value + ;; + -r|--registry) + ARTIFACTORY_DOCKER_REGISTRY="$2" + shift # past argument + shift # past value + ;; + -u|--artifactory-username) + ARTIFACTORY_USER="$2" + shift # past argument + shift # past value + ;; + -p|--artifactory-password) + ARTIFACTORY_PASSWORD="$2" + shift # past argument + shift # past value + ;; + *) # unknown option + POSITIONAL+=("$1") # save it in an array for later + shift # past argument + ;; +esac +done +set -- "${POSITIONAL[@]}" # restore positional parameters + +USAGE="push-docker-artifactory.sh -t|--docker-tag docker_tag -u|--artifactory-username ARTIFACTORY_USER -p|--artifactory-password ARTIFACTORY_PASSWORD" + +# shellcheck disable=SC2154 +if [[ -z "${docker_tag}" ]]; then + echo "docker_tag required." >&2 + echo "$USAGE" >&2 + exit 1 +fi + +# shellcheck disable=SC2154 +if [[ -z "${ARTIFACTORY_USER}" ]]; then + echo "ARTIFACTORY_USER required." >&2 + echo "$USAGE" >&2 + exit 1 +fi + +# shellcheck disable=SC2154 +if [[ -z "${ARTIFACTORY_PASSWORD}" ]]; then + echo "ARTIFACTORY_PASSWORD required." >&2 + echo "$USAGE" >&2 + exit 1 +fi + +echo "${ARTIFACTORY_PASSWORD}" | docker login --username "${ARTIFACTORY_USER}" --password-stdin "${ARTIFACTORY_DOCKER_REGISTRY}" +docker tag "${docker_tag}" "${ARTIFACTORY_DOCKER_REGISTRY}/${docker_tag}" +docker push "${ARTIFACTORY_DOCKER_REGISTRY}/${docker_tag}" \ No newline at end of file diff --git a/docker/push-docker-ecr.sh b/docker/push-docker-ecr.sh new file mode 100755 index 0000000..ae0f385 --- /dev/null +++ b/docker/push-docker-ecr.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash + +# This script is intended to be run by the CI/CD pipeline to push a docker tag previously built by build-docker.sh + +set -Eeo pipefail + +POSITIONAL=() +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + -t|--docker-tag) + docker_tag="$2" + shift # past argument + shift # past value + ;; + -v|--tf-venue) + tf_venue="$2" + case $tf_venue in + sit|uat|ops) ;; + *) + echo "tf_venue must be sit, uat, or ops" + exit 1;; + esac + shift # past argument + shift # past value + ;; + *) # unknown option + POSITIONAL+=("$1") # save it in an array for later + shift # past argument + ;; +esac +done +set -- "${POSITIONAL[@]}" # restore positional parameters + +USAGE="push-docker-ecr.sh -t|--docker-tag docker_tag -v|--tf-venue tf_venue" + +# shellcheck disable=SC2154 +if [[ -z "${tf_venue}" ]]; then + echo "tf_venue required. One of sit, uat, ops" >&2 + echo "$USAGE" >&2 + exit 1 +fi + +# shellcheck disable=SC2154 +if [[ -z "${docker_tag}" ]]; then + echo "docker_tag required." >&2 + echo "$USAGE" >&2 + exit 1 +fi + +set -u + +repositoryName=$(echo "${docker_tag}" | awk -F':' '{print $1}') +tf_profile="ngap-service-${tf_venue}" + +# Get the AWS Account ID for this venue/profile +# shellcheck disable=SC2154 +aws_acct=$(aws sts get-caller-identity --profile "$tf_profile" | python -c "import sys, json; print(json.load(sys.stdin)['Account'])") + +# Create repository if needed +aws ecr create-repository --repository-name "${repositoryName}" --profile "$tf_profile" || echo "No need to create, repository ${repositoryName} already exists" + +# Login to ECR +echo "aws ecr get-login-password --region us-west-2 --profile \"$tf_profile\" | docker login --username AWS --password-stdin \"$aws_acct\".dkr.ecr.us-west-2.amazonaws.com" +set +x +$(aws ecr get-login --no-include-email --region us-west-2 --profile "$tf_profile" 2> /dev/null) || \ + docker login --username AWS --password "$(aws ecr get-login-password --region us-west-2 --profile "$tf_profile")" "$aws_acct".dkr.ecr.us-west-2.amazonaws.com +set -x + +# Tag the image for this venue's ECR +docker tag "${docker_tag}" "$aws_acct".dkr.ecr.us-west-2.amazonaws.com/"${docker_tag}" + +# Push the tag +docker push "$aws_acct".dkr.ecr.us-west-2.amazonaws.com/"${docker_tag}"