Feature/cicd

.github/workflows/linter.yml

@@ -1,4 +1,4 @@
-name: Pylint
+name: Ruff
 
 on:
   push:
@@ -21,7 +21,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install pylint==3.1.0
-    - name: Running Pylint
+        pip install ruff
+    - name: Running Ruff Check
       run: |
-        pylint --rcfile=.pylintrc $(git ls-files '*.py')
+        ruff check $(git ls-files '*.py')

.github/workflows/tester.yml

@@ -0,0 +1,29 @@
+name: pytest
+
+on:
+  push:
+    branches:
+      - '**'
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.8"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pytest pandas tqdm
+        pip install -e .
+        pip install -r requirements.txt
+    - name: Running Tests
+      run: |
+        pytest tests/dfcx_scrapi/ -vv

.gitignore

@@ -1,6 +1,9 @@
 *.py[cod]
 *.sw[op]
 
+# specific files
+code-context.txt
+
 # cloud shell
 .theia
 

Makefile

@@ -11,11 +11,14 @@ pfreeze:
 	pip freeze > requirements.txt
 
 test:
-	pytest tests/dfcx_scrapi/core/$(f)
+	@if [ -n "$(v)" ]; then \
+		pytest tests/dfcx_scrapi/$(f) -vv; \
+	else \
+		pytest tests/dfcx_scrapi/$(f); \
+	fi
 
 lint:
-	pylint --rcfile=.pylintrc src/dfcx_scrapi/*
-	pylint --rcfile=.pylintrc tests/dfcx_scrapi/*
+	ruff check
 
 # just fix selected whitespace
 autofix-min-whitespace:
@@ -35,3 +38,6 @@ fix:
 build:
 	python3 -m build
 	pip uninstall dfcx-scrapi -y
+
+context-file:
+	find . -name "*.py" -print0 | xargs -0 -I {} sh -c 'echo "=== {} ==="; cat {}' > code-context.txt

examples/dfcx_agent_cicd/cicd_code/UAT/cloudbuild_deploy.yaml

@@ -0,0 +1,49 @@
+steps:
+
+  - id: SHAGCSCopy
+    name: gcr.io/google.com/cloudsdktool/cloud-sdk
+    #dir: 'set your path till the readme doc in the git'
+    entrypoint: /bin/bash
+    args:
+      - '-c'
+      - |
+        chmod 777 UAT/gcssha.sh
+        UAT/gcssha.sh $COMMIT_SHA
+
+  - id: deployagent
+    name: 'python:3.10'
+    #dir: 'set your path till the readme doc in the git'
+    entrypoint: /bin/bash
+    args:
+    - -c
+    - |
+      pip3 install -r UAT/requirements.txt
+      python3 -m UAT.deploy $COMMIT_SHA
+      echo $?
+
+  - id: CheckExitCode
+    name: 'gcr.io/cloud-builders/gcloud'
+    #dir: 'set your path till the readme doc in the git'
+    entrypoint: 'bash'
+    args:
+      - '-c'
+      - |
+        if [[ "$$BUILD_STATUS" -ne 0 ]]; then
+          echo "Stopping the build due to a previous failure."
+          exit 1
+        fi
+
+
+  - id: triggerproddeploy
+    name: gcr.io/google.com/cloudsdktool/cloud-sdk
+    #dir: 'set your path till the readme doc in the git'
+    entrypoint: /bin/bash
+    args:
+      - '-c'
+      - |
+        chmod 777 UAT/trigger.sh 
+        UAT/trigger.sh $LOCATION $COMMIT_SHA
+
+
+options:
+  logging: CLOUD_LOGGING_ONLY

examples/dfcx_agent_cicd/cicd_code/UAT/deploy.py

@@ -0,0 +1,71 @@
+""" UAT Deployment functions"""
+
+import sys
+import json
+import logging
+
+from shared.deployment import Deployment
+
+
+#from .shared.deployments import Deployment
+# logging config
+logging.basicConfig(
+    level=logging.INFO,
+    format="UAT: %(asctime)s %(levelname)-8s %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+
+def main(data):
+    """
+    Deploys and tests a Dialogflow CX agent in a UAT environment.
+
+    This function performs the following steps:
+
+    1. Initializes a Deployment object with the provided data.
+    2. Imports the agent to the specified UAT webhook environment.
+    3. Validates test cases.
+    4. Collects flow IDs.
+    5. Deletes versions based on count.
+    6. Cuts a new version.
+    7. Deploys the new version.
+    8. Updates the datastore with UAT information.
+
+    Args:
+        data: A dictionary containing configuration data, including the 'uat_webhook_env' key.
+    """
+
+    dep=Deployment(data)
+    # call the steps sequentially
+    dep.import_agent(webhookenv=data["uat_webhook_env"])
+    dep.test_case_validation()
+    dep.collect_flow_id()
+    dep.version_count_delete()
+    dep.version_cut()
+    dep.deploy_versions()
+    dep.datastore_update("uat")
+
+
+
+if __name__=="__main__":
+    # read env variables
+    with open("config.json" , encoding='utf-8') as config_file:
+        config = json.load(config_file)
+    SHA_ID=sys.argv[1]
+    obj=f"UAT/{config['agent_name']}/{SHA_ID}"
+    sha_gs_loc=(
+        f"gs://{config['bucket']}/UAT/{config['agent_name']}/{SHA_ID}"
+    )
+    logging.info("Agent location: %s" ,sha_gs_loc)
+    #adding additional variables to dict
+    config["sha_agent_gcs_location"]=sha_gs_loc
+    config["target_project_id"] = config["uat_project"]
+    config["target_environment_name"]=config["uat_env_deploy"]
+    with open("agent_artifacts/metadata.json" , encoding='utf-8') as metadata_file:
+        metadata = json.load(metadata_file)
+
+    config["source_flow_names"]=metadata["source_flow_names"]
+    config["updated_commit_message"]=metadata["updated_commit_message"]
+
+    # To execute steps in order
+    main(config)

examples/dfcx_agent_cicd/cicd_code/UAT/gcssha.sh

@@ -0,0 +1,21 @@
+
+# Set your GCS bucket name and destination directory
+apt-get update && apt-get install -y jq
+export GCS_BUCKET=$(jq -r .bucket config.json)
+export agent_name=$(jq -r .agent_name config.json)
+export DESTINATION_DIR="UAT/${agent_name}/"
+echo $DESTINATION_DIR
+# Create a local directory
+mkdir -p $1
+
+# Copy your two files to the local directory
+cp agent_artifacts/$agent_name $1
+cp agent_artifacts/metadata.json $1
+
+# Upload the local directory to GCS
+gsutil -m cp -r $1 "gs://$GCS_BUCKET/$DESTINATION_DIR"
+
+# Clean up the local directory if needed
+rm -r $1
+
+echo "Files copied and uploaded to GCS."

examples/dfcx_agent_cicd/cicd_code/UAT/requirements.txt

@@ -0,0 +1,3 @@
+dfcx-scrapi
+google-cloud-storage
+pandas

examples/dfcx_agent_cicd/cicd_code/UAT/trigger.sh

@@ -0,0 +1,22 @@
+echo $1
+apt-get update && apt-get install -y jq
+export devops_project_id=$(jq -r .devops_project config.json)
+export prod_project_id=$(jq -r .prod_project config.json)
+
+#Use below command to trigger the build if manual invokation is used. Since there is no secret , no extra charges
+
+export build_info=$(gcloud builds triggers run prodbuild --project=$devops_project_id --substitutions=_COMMIT_SHA=$2 --region=$1 --format=json)
+echo "devops prod triggerdone"
+
+
+#getting the trigger id of the above trigger
+
+export prod_build_id=$(echo "$build_info" | jq -r '.metadata.build.id')
+echo "build id returned back is"
+echo $prod_build_id
+
+
+#Trigger the build in prod project which is used for approval
+gcloud builds triggers run prodapprovebuild --project=$devops_project_id --substitutions=_APP_BUILD_ID=$prod_build_id --region=$1
+
+echo "prod project approve build triggered"

examples/dfcx_agent_cicd/cicd_code/approveprod/cloudbuild_appr.yaml

@@ -0,0 +1,36 @@
+steps:
+  - name: gcr.io/google.com/cloudsdktool/cloud-sdk
+    args:
+      - '-c'
+      - |
+        apt-get update && apt-get install -y jq
+        echo $BUILD_ID
+
+        export devopsprojecthere=$(jq -r .devops_project config.json)
+        export build_info=$(gcloud builds describe $BUILD_ID --region=us-central1 --format=json)
+        export approverhere=$(echo "$build_info" | jq -r '.approval.result.approverAccount')
+        export commenthere=$(echo "$build_info" | jq -r '.approval.result.comment')
+        export tokenhere=$(gcloud auth print-access-token)
+
+        echo $approverhere
+        echo $tokenhere
+
+        chmod 777 approveprod/trigger.sh
+
+        sed -i "s/tokenhere/$tokenhere/g" approveprod/trigger.sh
+        sed -i "s/approverhere/$approverhere/g" approveprod/trigger.sh
+        sed -i "s/devopsprojecthere/$devopsprojecthere/g" approveprod/trigger.sh
+        sed -i "s/commenthere/$commenthere/g" approveprod/trigger.sh
+        sed -i "s/appbuildhere/$_APP_BUILD_ID/g" approveprod/trigger.sh
+        cat approveprod/trigger.sh
+        approveprod/trigger.sh
+        echo $?
+        echo "prod build approved from code"
+
+        echo "error exit code"
+
+    id: triggerexportbuild
+    entrypoint: /bin/bash
+options:
+  logging: CLOUD_LOGGING_ONLY
+  dynamicSubstitutions: true

examples/dfcx_agent_cicd/cicd_code/approveprod/trigger.sh

@@ -0,0 +1,6 @@
+curl --request POST \
+    'https://cloudbuild.googleapis.com/v1/projects/devopsprojecthere/locations/us-central1/builds/appbuildhere:approve?access_token=tokenhere' \
+    --header 'Accept: application/json'\
+    --header 'Content-Type:application/json' --data \
+    '{"approvalResult":{"decision":"APPROVED","comment":"commenthere","approverAccount":"approverhere"}}' \
+    --compressed

examples/dfcx_agent_cicd/cicd_code/config.json

@@ -0,0 +1,16 @@
+{
+    "agent_name" : "carrental",
+    "dev_env_pull" : "ready to deploy",
+    "uat_env_deploy" : "ready to test",
+    "prod_env_deploy" :"deployed",
+    "devprodsyncenv" :"deployed",
+    "bucket": "dfcx_agent_cicd_export",
+    "dev_project": "yourprojectid",
+    "uat_project" : "yourprojectid",
+    "prod_project": "yourprojectid",
+    "devops_project": "yourprojectid",
+    "uat_webhook_env": "uat",
+    "prod_webhook_env": "prod",
+    "uat_engine_id" :"",
+    "prod_engine_id" :""
+}