Update full CICD pipeline

vectornguyen76 · Dec 1, 2024 · 8d10809 · 8d10809
1 parent e2f8fa1
commit 8d10809
Show file tree

Hide file tree

Showing 4 changed files with 34 additions and 333 deletions.
diff --git a/.github/workflows/production_pipeline.yml b/.github/workflows/production_pipeline.yml
@@ -4,333 +4,34 @@ on:
     branches:
       - master
 jobs:
-  build-push-image-search:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-          cache: "pip"
-
-      - name: Install Dependencies
-        run: pip install -r ./image_search/requirements.txt
-
-      - name: Run isort
-        run: isort --check-only --profile=black ./image_search/.
-
-      - name: Run black
-        run: black --check ./image_search/.
-
-      - name: Run flake8
-        run: flake8 --ignore=E501,W503,F401 ./image_search
-
-      # - name: Run Pylint
-      #   run: pylint ./image_search/*.py
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to Docker Hub
-        id: docker_hub_auth
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v4
-        with:
-          context: ./image_search
-          push: true
-          tags: ${{ secrets.DOCKERHUB_USERNAME }}/image_search:latest
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-  build-push-text-search:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-          cache: "pip"
-
-      - name: Install Dependencies
-        run: pip install -r ./text_search/requirements.txt
-
-      - name: Run isort
-        run: isort --check-only ./text_search/.
-
-      - name: Run black
-        run: black --check ./text_search/.
-
-      - name: Run flake8
-        run: flake8 --ignore=E501,W503,F401 ./text_search
-
-      # - name: Run Pylint
-      #   run: pylint ./image_search/*.py
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to Docker Hub
-        id: docker_hub_auth
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v4
-        with:
-          context: ./text_search
-          push: true
-          tags: ${{ secrets.DOCKERHUB_USERNAME }}/text_search:latest
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-  build-push-backend:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v3
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-          cache: "pip"
-
-      - name: Install Dependencies
-        run: pip install -r ./backend/requirements.txt
-
-      - name: Run isort
-        run: isort --check-only --profile=black ./backend/.
-
-      - name: Run black
-        run: black --check ./backend/.
-
-      - name: Run flake8
-        run: flake8 --ignore=E501,W503,F401 ./backend
-
-      # - name: Run Pylint
-      #   run: pylint ./backend/*.py
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to Docker Hub
-        id: docker_hub_auth
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v4
-        with:
-          context: ./backend
-          push: true
-          tags: ${{ secrets.DOCKERHUB_USERNAME }}/backend_search_engine:latest
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-  build-push-frontend:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to Docker Hub
-        id: docker_hub_auth
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_PASSWORD }}
-
-      - name: Add env variable to env file
-        run: |
-          echo GOOGLE_CLIENT_ID=${{ secrets.GOOGLE_CLIENT_ID }} >> ./frontend/.env
-          echo GOOGLE_CLIENT_SECRET=${{ secrets.GOOGLE_CLIENT_SECRET }} >> ./frontend/.env
-          echo NEXTAUTH_SECRET=${{ secrets.NEXTAUTH_SECRET }} >> ./frontend/.env
-
-      - name: Build and push frontend image
-        uses: docker/build-push-action@v4
-        with:
-          context: ./frontend
-          file: ./frontend/Dockerfile
-          push: true
-          tags: ${{ secrets.DOCKERHUB_USERNAME }}/frontend_search_engine:latest
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-  create-config-infrastructure:
-    runs-on: ubuntu-latest
+  ci:
+    uses: ./.github/workflows/ci.yml
+  cd:
     needs:
-      - build-push-image-search
-      - build-push-text-search
-      - build-push-backend
-      - build-push-frontend
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v3
-
-      - name: Declare variables
-        shell: bash
-        run: |
-          echo "SHA_SHORT=$(git rev-parse --short "$GITHUB_SHA")" >> "$GITHUB_ENV"
-          echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> "$GITHUB_ENV"
-
-      - name: Configure AWS credentials
-        id: creds
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-1
-
-      - name: Deploy to AWS CloudFormation
-        uses: aws-actions/aws-cloudformation-github-deploy@v1
-        with:
-          name: search-engine-prod-${{ env.SHA_SHORT }}
-          template: ./.github/workflows/cloudformations/server.yml
-          parameter-overrides: "file:///${{ github.workspace }}/.github/workflows/cloudformations/server-parameters.json"
-          tags: ${{ vars.TAGS }}
-
-      - name: Get Public DNS Server
-        run: |
-          # Create file
-          backend_public_dns=search-engine-prod-${{ env.SHA_SHORT }}-PublicDNS
-          # Pull the export value
-          host=$(aws cloudformation list-exports \
-          --query "Exports[?Name==\`$backend_public_dns\`].Value" \
-          --no-paginate --output text)
-
-          echo $host
-          # Append the DNS to the inventory file
-          echo $host >> $(eval echo "./.github/workflows/ansible/hosts")
-
-          cat ./.github/workflows/ansible/hosts
-
-      - name: Zip artifact files
-        uses: montudor/action-zip@v1
-        with:
-          args: zip -qq -r artifact.zip .
-
-      - name: Create files forlder in ansible
-        run: mkdir -p ./.github/workflows/ansible/roles/deploy/files
-
-      - name: Copy file
-        uses: canastro/copy-file-action@master
-        with:
-          source: "artifact.zip"
-          target: "./.github/workflows/ansible/roles/deploy/files/artifact.zip"
-
-      - name: Run playbook
-        uses: dawidd6/action-ansible-playbook@v2
-        with:
-          playbook: deploy_applications.yml
-          directory: ./.github/workflows/ansible
-          key: ${{secrets.SSH_PRIVATE_KEY}}
-          options: |
-            --inventory ./hosts
-
-      - name: Remove stack on fail
-        if: failure()
-        run: |
-          echo search-engine-prod-${{ env.SHA_SHORT }}
-          # Get stack id for the delete_stack waiter
-          stack_info=$(aws cloudformation describe-stacks --stack-name search-engine-prod-${{ env.SHA_SHORT }} --query "Stacks[*] | [0].StackId" 2>&1)
-          if echo $stack_info | grep 'does not exist' > /dev/null
-          then
-            echo "Stack does not exist."
-            echo $stack_info
-            exit 0
-          fi
-          if echo $stack_info | grep 'ValidationError' > /dev/null
-          then
-            echo $stack_info
-            exit 1
-          else
-            aws cloudformation delete-stack --stack-name search-engine-prod-${{ env.SHA_SHORT }}
-            echo $stack_info
-            aws cloudformation wait stack-delete-complete --stack-name search-engine-prod-${{ env.SHA_SHORT }}
-            exit 0
-          fi
-
-  clean-up:
-    runs-on: ubuntu-latest
+      - ci
+    uses: ./.github/workflows/cd.yml
+    with:
+      stack_name: search-engine-prod
+      domain_name: search.vectornguyen.com
+      hosted_zone_id: Z02762033GQ721WO21JL8
+    secrets:
+      GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }}
+      GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }}
+      NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET }}
+
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+  rollback:
     needs:
-      - create-config-infrastructure
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v3
-
-      - name: Declare some variables
-        shell: bash
-        run: |
-          echo "SHA_SHORT=$(git rev-parse --short "$GITHUB_SHA")" >> "$GITHUB_ENV"
-          echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> "$GITHUB_ENV"
-
-      - name: Configure AWS credentials
-        id: creds
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-1
-
-      - name: Fetch stacks and save the old stack name
-        run: |
-          # Fetch the stack names
-          export STACKS=(
-            $(aws cloudformation list-stacks \
-                --query "StackSummaries[*].StackName" \
-                --no-paginate --output text \
-                --stack-status-filter CREATE_COMPLETE UPDATE_COMPLETE
-            )
-          )
-          for stack in "${STACKS[@]}"; do
-            if [[ ! "$stack" =~ "${{ env.SHA_SHORT }}" ]] && [[ "$stack" =~ "search-engine-prod" ]]; then
-              echo "DESTROY_STACK=$stack" >> "$GITHUB_ENV"
-            fi
-          done
-
-      - name: Remove the search engine infrastructure
-        run: |
-          # Check if DESTROY_STACK is not set
-          if [ -z "${{ env.DESTROY_STACK }}" ]; then
-            echo "DESTROY_STACK is not set"
-            exit 0
-          else
-            echo "DESTROY_STACK is set to ${{ env.DESTROY_STACK }}"
-          fi
-
-          # Get stack id for the delete_stack waiter
-          stack_info=$(aws cloudformation describe-stacks --stack-name ${{ env.DESTROY_STACK }} --query "Stacks[*] | [0].StackId" 2>&1)
-          if echo $stack_info | grep 'does not exist' > /dev/null
-          then
-            echo "Stack does not exist."
-            echo $stack_info
-            exit 0
-          fi
-          if echo $stack_info | grep 'ValidationError' > /dev/null
-          then
-            echo $stack_info
-            exit 1
-          else
-            aws cloudformation delete-stack --stack-name ${{ env.DESTROY_STACK }}
-            echo $stack_info
-            aws cloudformation wait stack-delete-complete --stack-name ${{ env.DESTROY_STACK }}
-            exit 0
-          fi
+      - cd
+    uses: ./.github/workflows/rollback.yml
+    if: failure()
+    with:
+      stack_name: search-engine-prod
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
diff --git a/backend/src/image_search/service.py b/backend/src/image_search/service.py
@@ -20,7 +20,7 @@ async def image_search(
             # Upload the image to the /search-image-qdrant endpoint
             response = await client.post(
                 # f"{settings.IMAGE_SEARCH_URL}/search-image-qdrant", files=files
-                f"{settings.IMAGE_SEARCH_URL}/search-image",
+                f"{settings.IMAGE_SEARCH_URL}/search-image-faiss",
                 files=files,
             )
 

diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -64,7 +64,7 @@ services:
         condition: service_healthy
 
   image_search:
-    container_name: image-search-container
+    container_name: image_search
     image: vectornguyen76/image_search
     build:
       context: ./image_search
@@ -80,7 +80,7 @@ services:
       - qdrant_db
 
   text_search:
-    container_name: text-search-container
+    container_name: text_search
     image: vectornguyen76/text_search
     build:
       context: ./text_search