fix: fix use wrong volume for kafka and use shared tf file

Signed-off-by: Kami Wan <[email protected]>

.github/actions/prepare-vs-shared/action.yml

@@ -76,31 +76,31 @@ runs:
       uses: hashicorp/setup-terraform@v3
 
     - name: Initialize terraform
-      working-directory: driver-${{ inputs.streaming_provider }}/deploy/${{ inputs.cloud_provider }}
+      working-directory: driver-shared/deploy/${{ inputs.cloud_provider }}
       shell: bash
       run: terraform init
 
     - name: Uninstall Cloud Infra
-      working-directory: driver-${{ inputs.streaming_provider }}/deploy/${{ inputs.cloud_provider }}
+      working-directory: driver-shared/deploy/${{ inputs.cloud_provider }}
       shell: bash
       run: |
         if [ "${{ inputs.uninstall }}" == "true" ]; then
-          terraform destroy --auto-approve -var-file var.tfvars
+          terraform destroy --auto-approve -var-file $GITHUB_WORKSPACE/driver-${{ inputs.streaming_provider }}/deploy/${{ inputs.cloud_provider }}/var.tfvars
         fi     
 
 
     - name: Terraform Plan
-      working-directory: driver-${{ inputs.streaming_provider }}/deploy/${{ inputs.cloud_provider }}
+      working-directory: driver-shared/deploy/${{ inputs.cloud_provider }}
       shell: bash
-      run: terraform plan -var-file var.tfvars
+      run: terraform plan -var-file $GITHUB_WORKSPACE/driver-${{ inputs.streaming_provider }}/deploy/${{ inputs.cloud_provider }}/var.tfvars
 
 
     - name: Apply terraform
-      working-directory: driver-${{ inputs.streaming_provider }}/deploy/${{ inputs.cloud_provider }}
+      working-directory: driver-shared/deploy/${{ inputs.cloud_provider }}
       shell: bash
       run: |
         if [ "${{ inputs.uninstall }}" = "false" ]; then
-          terraform apply --auto-approve -var-file var.tfvars
+          terraform apply --auto-approve -var-file $GITHUB_WORKSPACE/driver-${{ inputs.streaming_provider }}/deploy/${{ inputs.cloud_provider }}/var.tfvars
         fi
 
 
@@ -123,25 +123,20 @@ runs:
 
 
     - name: Install Streaming Cluster
-      working-directory: driver-${{ inputs.streaming_provider }}/deploy/${{ inputs.cloud_provider }}
+      working-directory: driver-shared/deploy/${{ inputs.cloud_provider }}
       shell: bash
       run: |
         if [[ "${{ inputs.uninstall }}" = "false" && "${{ inputs.execute_benchmark }}" = "false" ]]; then
-          if [ "${{ inputs.streaming_provider }}" = "automq" ]; then
-              echo "Install AutoMQ"
-              ansible-playbook deploy.yaml -i hosts.ini
-          elif [ "${{ inputs.streaming_provider }}" = "kafka" ]; then
               echo "Install Apache Kafka"
               wget https://github.com/adammck/terraform-inventory/releases/download/v0.10/terraform-inventory_v0.10_linux_amd64.zip
               unzip terraform-inventory_v0.10_linux_amd64.zip
               mv terraform-inventory /usr/local/bin
-              ansible-playbook   --user ubuntu --inventory `which terraform-inventory` deploy.yaml 
-          fi
+              ansible-playbook   --user ubuntu -i hosts.ini --inventory `which terraform-inventory` $GITHUB_WORKSPACE/driver-${{ inputs.streaming_provider }}/deploy/${{ inputs.cloud_provider }}/deploy.yaml 
         fi
       
 
     - name: Execute Benchmark
-      working-directory: driver-${{ inputs.streaming_provider }}/deploy/${{ inputs.cloud_provider }}
+      working-directory: driver-shared/deploy/${{ inputs.cloud_provider }}
       shell: bash
       run: |
         if [ "${{ inputs.execute_benchmark }}" = "true" ]; then

driver-automq/deploy/aws-cn/var.tfvars

@@ -1,8 +1,5 @@
-public_key_path = "~/.ssh/automq_aws-cn.pub"
-region          = "cn-northwest-1"
-az              = ["cn-northwest-1a", "cn-northwest-1b"]
-
-ami             = "ami-04c77a27ae5156100" // Canonical, Ubuntu, 22.04 LTS, amd64 jammy image build on 2023-03-03
+// Only the following vars can be customized
+ami             = "ami-04c77a27ae5156100" // Ubuntu 22.04 LTS for x86_64
 user            = "ubuntu"
 
 instance_type = {
@@ -17,14 +14,9 @@ instance_cnt = {
   "client"              = 2
 }
 
-monitoring = true
-spot = false
-
 ebs_volume_type = "gp3"
 ebs_volume_size = 10
 ebs_iops = 3000
+ebs_throughput = 125
 
-access_key = "${AUTOMQ_ACCESS_KEY}"
-secret_key = "${AUTOMQ_SECRET_KEY}"
 
-aws_cn = true

driver-kafka/deploy/aws-cn/deploy.yaml

@@ -94,19 +94,15 @@
     - name: Format disks
       filesystem:
         fstype: ext4
-        dev: '{{ item }}'
-      with_items:
-        ## aws default raw data disk name is /dev/nvme1n1
-        - '/dev/nvme1n1'
+        dev: '{{ hostvars[inventory_hostname]["data_volume"] }}'
+
     - name: Mount disks
       mount:
-        path: "{{ item.path }}"
-        src: "{{ item.src }}"
+        path: "{{ base_data_path }}"
+        src: "{{ hostvars[inventory_hostname]['data_volume'] }}"
         fstype: ext4
         opts: defaults,noatime,nodiscard,data=writeback
         state: mounted
-      with_items:
-        - { path: "{{ base_data_path }}", src: "/dev/nvme1n1" }
 
 - name: Kafka general setup
   tags: [prepare]
@@ -127,7 +123,7 @@
         - set_fact:
             quorumServers: "{{ quorumServers[:-1] }}"
             kafkaVersion: "3.6.0"
-            KAFKA_CLUSTER_ID: "XPufKTN9T1SpE82LDKwXwA"
+            KAFKA_CLUSTER_ID: "{{ hostvars[inventory_hostname]['cluster_id'] }}"
             flushScheduleInterval: "9223372036854775807"
     - name: Download Kafka package
       unarchive:

driver-kafka/deploy/aws-cn/var.tfvars

@@ -1,8 +1,5 @@
-public_key_path = "~/.ssh/kafka_aws-cn.pub"
-region          = "cn-northwest-1"
-az              = ["cn-northwest-1a", "cn-northwest-1b"]
+// Only the following vars can be customized
 ami             = "ami-04c77a27ae5156100" // Ubuntu 22.04 LTS for x86_64
-// ami = "ami-08133f9f7ea98ef23" Ubuntu 22.04 LTS for arm64
 user            = "ubuntu"
 
 instance_type = {
@@ -17,8 +14,6 @@ instance_cnt = {
   "client"              = 2
 }
 
-monitoring = true
-
 ebs_volume_type = "gp3"
 ebs_volume_size = 2048
 ebs_iops = 3000