diff --git a/.github/workflows/production_pipeline.yml b/.github/workflows/production_pipeline.yml index 93223d4..801d138 100644 --- a/.github/workflows/production_pipeline.yml +++ b/.github/workflows/production_pipeline.yml @@ -4,333 +4,34 @@ on: branches: - master jobs: - build-push-image-search: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.9 - cache: "pip" - - - name: Install Dependencies - run: pip install -r ./image_search/requirements.txt - - - name: Run isort - run: isort --check-only --profile=black ./image_search/. - - - name: Run black - run: black --check ./image_search/. - - - name: Run flake8 - run: flake8 --ignore=E501,W503,F401 ./image_search - - # - name: Run Pylint - # run: pylint ./image_search/*.py - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to Docker Hub - id: docker_hub_auth - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: ./image_search - push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/image_search:latest - cache-from: type=gha - cache-to: type=gha,mode=max - - build-push-text-search: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.9 - cache: "pip" - - - name: Install Dependencies - run: pip install -r ./text_search/requirements.txt - - - name: Run isort - run: isort --check-only ./text_search/. - - - name: Run black - run: black --check ./text_search/. - - - name: Run flake8 - run: flake8 --ignore=E501,W503,F401 ./text_search - - # - name: Run Pylint - # run: pylint ./image_search/*.py - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to Docker Hub - id: docker_hub_auth - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: ./text_search - push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/text_search:latest - cache-from: type=gha - cache-to: type=gha,mode=max - - build-push-backend: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.10" - cache: "pip" - - - name: Install Dependencies - run: pip install -r ./backend/requirements.txt - - - name: Run isort - run: isort --check-only --profile=black ./backend/. - - - name: Run black - run: black --check ./backend/. - - - name: Run flake8 - run: flake8 --ignore=E501,W503,F401 ./backend - - # - name: Run Pylint - # run: pylint ./backend/*.py - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to Docker Hub - id: docker_hub_auth - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Build and push - uses: docker/build-push-action@v4 - with: - context: ./backend - push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/backend_search_engine:latest - cache-from: type=gha - cache-to: type=gha,mode=max - - build-push-frontend: - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to Docker Hub - id: docker_hub_auth - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - - - name: Add env variable to env file - run: | - echo GOOGLE_CLIENT_ID=${{ secrets.GOOGLE_CLIENT_ID }} >> ./frontend/.env - echo GOOGLE_CLIENT_SECRET=${{ secrets.GOOGLE_CLIENT_SECRET }} >> ./frontend/.env - echo NEXTAUTH_SECRET=${{ secrets.NEXTAUTH_SECRET }} >> ./frontend/.env - - - name: Build and push frontend image - uses: docker/build-push-action@v4 - with: - context: ./frontend - file: ./frontend/Dockerfile - push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/frontend_search_engine:latest - cache-from: type=gha - cache-to: type=gha,mode=max - - create-config-infrastructure: - runs-on: ubuntu-latest + ci: + uses: ./.github/workflows/ci.yml + cd: needs: - - build-push-image-search - - build-push-text-search - - build-push-backend - - build-push-frontend - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - - - name: Declare variables - shell: bash - run: | - echo "SHA_SHORT=$(git rev-parse --short "$GITHUB_SHA")" >> "$GITHUB_ENV" - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> "$GITHUB_ENV" - - - name: Configure AWS credentials - id: creds - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Deploy to AWS CloudFormation - uses: aws-actions/aws-cloudformation-github-deploy@v1 - with: - name: search-engine-prod-${{ env.SHA_SHORT }} - template: ./.github/workflows/cloudformations/server.yml - parameter-overrides: "file:///${{ github.workspace }}/.github/workflows/cloudformations/server-parameters.json" - tags: ${{ vars.TAGS }} - - - name: Get Public DNS Server - run: | - # Create file - backend_public_dns=search-engine-prod-${{ env.SHA_SHORT }}-PublicDNS - # Pull the export value - host=$(aws cloudformation list-exports \ - --query "Exports[?Name==\`$backend_public_dns\`].Value" \ - --no-paginate --output text) - - echo $host - # Append the DNS to the inventory file - echo $host >> $(eval echo "./.github/workflows/ansible/hosts") - - cat ./.github/workflows/ansible/hosts - - - name: Zip artifact files - uses: montudor/action-zip@v1 - with: - args: zip -qq -r artifact.zip . - - - name: Create files forlder in ansible - run: mkdir -p ./.github/workflows/ansible/roles/deploy/files - - - name: Copy file - uses: canastro/copy-file-action@master - with: - source: "artifact.zip" - target: "./.github/workflows/ansible/roles/deploy/files/artifact.zip" - - - name: Run playbook - uses: dawidd6/action-ansible-playbook@v2 - with: - playbook: deploy_applications.yml - directory: ./.github/workflows/ansible - key: ${{secrets.SSH_PRIVATE_KEY}} - options: | - --inventory ./hosts - - - name: Remove stack on fail - if: failure() - run: | - echo search-engine-prod-${{ env.SHA_SHORT }} - # Get stack id for the delete_stack waiter - stack_info=$(aws cloudformation describe-stacks --stack-name search-engine-prod-${{ env.SHA_SHORT }} --query "Stacks[*] | [0].StackId" 2>&1) - if echo $stack_info | grep 'does not exist' > /dev/null - then - echo "Stack does not exist." - echo $stack_info - exit 0 - fi - if echo $stack_info | grep 'ValidationError' > /dev/null - then - echo $stack_info - exit 1 - else - aws cloudformation delete-stack --stack-name search-engine-prod-${{ env.SHA_SHORT }} - echo $stack_info - aws cloudformation wait stack-delete-complete --stack-name search-engine-prod-${{ env.SHA_SHORT }} - exit 0 - fi - - clean-up: - runs-on: ubuntu-latest + - ci + uses: ./.github/workflows/cd.yml + with: + stack_name: search-engine-prod + domain_name: search.vectornguyen.com + hosted_zone_id: Z02762033GQ721WO21JL8 + secrets: + GOOGLE_CLIENT_ID: ${{ secrets.GOOGLE_CLIENT_ID }} + GOOGLE_CLIENT_SECRET: ${{ secrets.GOOGLE_CLIENT_SECRET }} + NEXTAUTH_SECRET: ${{ secrets.NEXTAUTH_SECRET }} + + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + + rollback: needs: - - create-config-infrastructure - steps: - - name: Checkout Repository - uses: actions/checkout@v3 - - - name: Declare some variables - shell: bash - run: | - echo "SHA_SHORT=$(git rev-parse --short "$GITHUB_SHA")" >> "$GITHUB_ENV" - echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> "$GITHUB_ENV" - - - name: Configure AWS credentials - id: creds - uses: aws-actions/configure-aws-credentials@v1 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Fetch stacks and save the old stack name - run: | - # Fetch the stack names - export STACKS=( - $(aws cloudformation list-stacks \ - --query "StackSummaries[*].StackName" \ - --no-paginate --output text \ - --stack-status-filter CREATE_COMPLETE UPDATE_COMPLETE - ) - ) - for stack in "${STACKS[@]}"; do - if [[ ! "$stack" =~ "${{ env.SHA_SHORT }}" ]] && [[ "$stack" =~ "search-engine-prod" ]]; then - echo "DESTROY_STACK=$stack" >> "$GITHUB_ENV" - fi - done - - - name: Remove the search engine infrastructure - run: | - # Check if DESTROY_STACK is not set - if [ -z "${{ env.DESTROY_STACK }}" ]; then - echo "DESTROY_STACK is not set" - exit 0 - else - echo "DESTROY_STACK is set to ${{ env.DESTROY_STACK }}" - fi - - # Get stack id for the delete_stack waiter - stack_info=$(aws cloudformation describe-stacks --stack-name ${{ env.DESTROY_STACK }} --query "Stacks[*] | [0].StackId" 2>&1) - if echo $stack_info | grep 'does not exist' > /dev/null - then - echo "Stack does not exist." - echo $stack_info - exit 0 - fi - if echo $stack_info | grep 'ValidationError' > /dev/null - then - echo $stack_info - exit 1 - else - aws cloudformation delete-stack --stack-name ${{ env.DESTROY_STACK }} - echo $stack_info - aws cloudformation wait stack-delete-complete --stack-name ${{ env.DESTROY_STACK }} - exit 0 - fi + - cd + uses: ./.github/workflows/rollback.yml + if: failure() + with: + stack_name: search-engine-prod + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/backend/src/image_search/service.py b/backend/src/image_search/service.py index 456c8ea..09da11f 100644 --- a/backend/src/image_search/service.py +++ b/backend/src/image_search/service.py @@ -20,7 +20,7 @@ async def image_search( # Upload the image to the /search-image-qdrant endpoint response = await client.post( # f"{settings.IMAGE_SEARCH_URL}/search-image-qdrant", files=files - f"{settings.IMAGE_SEARCH_URL}/search-image", + f"{settings.IMAGE_SEARCH_URL}/search-image-faiss", files=files, ) diff --git a/docker-compose.yaml b/docker-compose.yaml index 7b41b9d..146f6a7 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -64,7 +64,7 @@ services: condition: service_healthy image_search: - container_name: image-search-container + container_name: image_search image: vectornguyen76/image_search build: context: ./image_search @@ -80,7 +80,7 @@ services: - qdrant_db text_search: - container_name: text-search-container + container_name: text_search image: vectornguyen76/text_search build: context: ./text_search diff --git a/image_search/entrypoint.sh b/image_search/entrypoint.sh index 07fee29..ac3aeb3 100644 --- a/image_search/entrypoint.sh +++ b/image_search/entrypoint.sh @@ -3,8 +3,8 @@ echo "Start ingest data to Faiss search..." python faiss_ingest.py -echo "Start ingest data to Qdrant search..." -python qdrant_ingest.py +# echo "Start ingest data to Qdrant search..." +# python qdrant_ingest.py echo "Run app with uvicorn server..." uvicorn app:app --port 7000 --host 0.0.0.0 --workers 1