Merge branch 'main' into priyanshi/callhome-error

yugabyte · Feb 12, 2025 · 8867d54 · 8867d54
2 parents 815b8e7 + 1aa4a47
commit 8867d54
Show file tree

Hide file tree

Showing 61 changed files with 12,682 additions and 5,219 deletions.
diff --git a/.github/workflows/issue-tests.yml b/.github/workflows/issue-tests.yml
@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: [2.23.1.0-b220, 2024.1.3.1-b8, 2024.2.0.0-b145, 2.20.8.0-b53, 2.25.0.0-b489]
+        version: [2.23.1.0-b220, 2024.1.4.0-b108, 2024.2.1.0-b185, 2.20.9.0-b47, 2.25.0.0-b489]
     env:
       YB_VERSION: ${{ matrix.version }}
       YB_CONN_STR: "postgres://yugabyte:[email protected]:5433/yugabyte"

diff --git a/.github/workflows/misc-migtests.yml b/.github/workflows/misc-migtests.yml
@@ -92,12 +92,12 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{ secrets.RAHULB_S3_SECRET_ACCESS_KEY }}
         if: ${{ !cancelled() }}
         run: |
-          versions=("2024.2.0.0-b145" "2.20.8.0-b53" "2024.1.3.1-b8" "2.23.1.0-b220")
+          versions=("2024.2.1.0-b185" "2.20.9.0-b47" "2024.1.4.0-b108" "2.23.1.0-b220")
           for version in "${versions[@]}"; do
             echo "Running tests on version $version"
 
             echo "Start YugabyteDB cluster"
-            docker run -d --name yugabytedb-$version \
+            docker run -d --name yugabytedb-$version --hostname yb-cluster-test-$version\
             -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
             yugabytedb/yugabyte:$version \
             bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false 
@@ -109,10 +109,10 @@ jobs:
             echo "Create YugabyteDB user"
             ./migtests/scripts/yugabytedb/create_yb_user
 
-            echo "Enable yb-tserver-n1 and yb-master-n1 name resolution"
-            echo "127.0.0.1    yb-tserver-n1" | sudo tee -a /etc/hosts
-            echo "127.0.0.1    yb-master-n1" | sudo tee -a /etc/hosts
-            psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
+            echo "Enable yb-cluster-test-$version name resolution"
+            docker_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' yugabytedb-$version)
+            echo "$docker_ip	yb-cluster-test-$version" | sudo tee -a /etc/hosts
+            psql "postgresql://yugabyte@yb-cluster-test-$version:5433/yugabyte" -c "SELECT version();"
 
             echo "Setup the gcp credentials"
             migtests/scripts/gcs/create_gcs_credentials_file

diff --git a/.github/workflows/mysql-migtests.yml b/.github/workflows/mysql-migtests.yml
@@ -10,7 +10,7 @@ jobs:
   run-mysql-migration-tests:
     strategy:
       matrix:
-        version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220]
+        version: [2024.2.1.0-b185, 2.20.9.0-b47, 2024.1.4.0-b108, 2.23.1.0-b220]
         BETA_FAST_DATA_EXPORT: [0, 1]
     env:
       BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }}
@@ -60,7 +60,7 @@ jobs:
 
       - name: Start YugabyteDB cluster
         run: |
-          docker run -d --name yugabytedb \
+          docker run -d --name yugabytedb --hostname yb-cluster-test\
             -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
             yugabytedb/yugabyte:${{ matrix.version }} \
             bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false 
@@ -74,10 +74,11 @@ jobs:
         run: |
           ./migtests/scripts/yugabytedb/create_yb_user
 
-      - name: Enable yb-tserver-n1 name resolution
+      - name: Enable yb-cluster-test name resolution
         run: |
-          echo "127.0.0.1	yb-tserver-n1" | sudo tee -a /etc/hosts
-          psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
+          docker_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' yugabytedb)
+          echo "$docker_ip	yb-cluster-test" | sudo tee -a /etc/hosts
+          psql "postgresql://yugabyte@yb-cluster-test:5433/yugabyte" -c "SELECT version();"
 
       - name: "TEST: mysql-table-list-flags-test (table-list and exclude-table-list)"
         if: ${{ !cancelled() }}

diff --git a/.github/workflows/pg-13-migtests.yml b/.github/workflows/pg-13-migtests.yml
@@ -10,7 +10,7 @@ jobs:
   run-pg-13-migration-tests:
     strategy:
       matrix:
-        version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220]
+        version: [2024.2.1.0-b185, 2.20.9.0-b47, 2024.1.4.0-b108, 2.23.1.0-b220]
         BETA_FAST_DATA_EXPORT: [0, 1]
         test_group:
           - offline
@@ -89,7 +89,7 @@ jobs:
 
       - name: Start YugabyteDB cluster
         run: |
-          docker run -d --name yugabytedb \
+          docker run -d --name yugabytedb --hostname yb-cluster-test\
             -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
             yugabytedb/yugabyte:${{ matrix.version }} \
             bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false 
@@ -103,11 +103,11 @@ jobs:
         run: |
           ./migtests/scripts/yugabytedb/create_yb_user
 
-      - name: Enable yb-tserver-n1 and yb-master-n1 name resolution
+      - name: Enable yb-cluster-test name resolution
         run: |
-          echo "127.0.0.1	yb-tserver-n1" | sudo tee -a /etc/hosts
-          echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts
-          psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
+          docker_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' yugabytedb)
+          echo "$docker_ip	yb-cluster-test" | sudo tee -a /etc/hosts
+          psql "postgresql://yugabyte@yb-cluster-test:5433/yugabyte" -c "SELECT version();"
 
       - name: "TEST: pg-table-list-flags-test (table-list and exclude-table-list)"
         if: ${{ !cancelled() && matrix.test_group == 'offline' }}

diff --git a/.github/workflows/pg-17-migtests.yml b/.github/workflows/pg-17-migtests.yml
@@ -10,7 +10,7 @@ jobs:
   run-pg-17-migration-tests:
     strategy:
       matrix:
-        version: [2024.2.0.0-b145, 2.20.8.0-b53, 2024.1.3.1-b8, 2.23.1.0-b220]
+        version: [2024.2.1.0-b185, 2.20.9.0-b47, 2024.1.4.0-b108, 2.23.1.0-b220]
         BETA_FAST_DATA_EXPORT: [0, 1]
         test_group:
           - offline
@@ -89,7 +89,7 @@ jobs:
 
       - name: Start YugabyteDB cluster
         run: |
-          docker run -d --name yugabytedb \
+          docker run -d --name yugabytedb --hostname yb-cluster-test\
             -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
             yugabytedb/yugabyte:${{ matrix.version }} \
             bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false 
@@ -103,11 +103,11 @@ jobs:
         run: |
           ./migtests/scripts/yugabytedb/create_yb_user
 
-      - name: Enable yb-tserver-n1 and yb-master-n1 name resolution
+      - name: Enable yb-cluster-test name resolution
         run: |
-          echo "127.0.0.1	yb-tserver-n1" | sudo tee -a /etc/hosts
-          echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts
-          psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
+          docker_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' yugabytedb)
+          echo "$docker_ip	yb-cluster-test" | sudo tee -a /etc/hosts
+          psql "postgresql://yugabyte@yb-cluster-test:5433/yugabyte" -c "SELECT version();"
 
       - name: "TEST: PG sample schemas (sakila)"
         if: ${{ !cancelled() && matrix.test_group == 'offline' }}

diff --git a/.github/workflows/pg-9-migtests.yml b/.github/workflows/pg-9-migtests.yml
@@ -10,7 +10,7 @@ jobs:
   run-pg-9-migration-tests:
     strategy:
       matrix:
-        version: [2024.2.0.0-b145]
+        version: [2024.2.1.0-b185]
         BETA_FAST_DATA_EXPORT: [0, 1]
     env:
       BETA_FAST_DATA_EXPORT: ${{ matrix.BETA_FAST_DATA_EXPORT }}
@@ -82,7 +82,7 @@ jobs:
 
       - name: Start YugabyteDB cluster
         run: |
-          docker run -d --name yugabytedb \
+          docker run -d --name yugabytedb --hostname yb-cluster-test\
             -p7000:7000 -p9000:9000 -p15433:15433 -p5433:5433 -p9042:9042 \
             yugabytedb/yugabyte:${{ matrix.version }} \
             bin/yugabyted start --tserver_flags="ysql_hba_conf_csv={host all yugabyte all trust,host all all all md5}" --background=false --ui=false 
@@ -96,11 +96,11 @@ jobs:
         run: |
           ./migtests/scripts/yugabytedb/create_yb_user
 
-      - name: Enable yb-tserver-n1 and yb-master-n1 name resolution
+      - name: Enable yb-cluster-test name resolution
         run: |
-          echo "127.0.0.1	yb-tserver-n1" | sudo tee -a /etc/hosts
-          echo "127.0.0.1 yb-master-n1" | sudo tee -a /etc/hosts
-          psql "postgresql://yugabyte@yb-tserver-n1:5433/yugabyte" -c "SELECT version();"
+          docker_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' yugabytedb)
+          echo "$docker_ip	yb-cluster-test" | sudo tee -a /etc/hosts
+          psql "postgresql://yugabyte@yb-cluster-test:5433/yugabyte" -c "SELECT version();"
 
       - name: "TEST: pg-case-sensitivity-single-table"
         if: ${{ !cancelled() }}

diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh
@@ -874,7 +874,7 @@ normalize_json() {
     local input_file="$1"
     local output_file="$2"
     local temp_file="/tmp/temp_file.json"
-	# local temp_file2="/tmp/temp_file2.json"
+	local temp_file2="/tmp/temp_file2.json"
 
     # Normalize JSON with jq; use --sort-keys to avoid the need to keep the same sequence of keys in expected vs actual json
     jq --sort-keys 'walk(
@@ -909,23 +909,22 @@ normalize_json() {
         end
     )' "$input_file" > "$temp_file"
 
-	# Sample code: how to ignore nested fields with a specific parent in a json
-	# TODO: For fields like SqlStatement or others we can more explicit regarding changing them
-    # jq --sort-keys 'walk(
-    #     if type == "object" and (.UnsupportedFeatures?|type) == "array" then
-	# 		.UnsupportedFeatures |= map(
-	# 			.FeatureName? = "IGNORED"
-	# 		)
-    #     else
-	# 		.
-    #     end
-    # )' "$temp_file" > "$temp_file2"
+	# Second pass: for AssessmentIssue objects, if Category is "migration_caveats" and the Type is one
+    # of the two specified values, then set ObjectName to "IGNORED".
+    jq 'walk(
+        if type == "object" and
+			.Category == "migration_caveats" and
+			(.Type == "UNSUPPORTED_DATATYPE_LIVE_MIGRATION" or .Type == "UNSUPPORTED_DATATYPE_LIVE_MIGRATION_WITH_FF_FB")
+        then .ObjectName = "IGNORED"
+        else .
+        end
+    )' "$temp_file" > "$temp_file2"
 
     # Remove unwanted lines
-    sed -i '/Review and manually import.*uncategorized.sql/d' "$temp_file"
+    sed -i '/Review and manually import.*uncategorized.sql/d' "$temp_file2"
 
     # Move cleaned file to output
-    mv "$temp_file" "$output_file"
+    mv "$temp_file2" "$output_file"
 }
 
 
@@ -943,6 +942,11 @@ compare_sql_files() {
     sed -i -E 's#could not open extension control file ".*/(postgis\.control)"#could not open extension control file "PATH_PLACEHOLDER/\1"#g' "$normalized_file1"
     sed -i -E 's#could not open extension control file ".*/(postgis\.control)"#could not open extension control file "PATH_PLACEHOLDER/\1"#g' "$normalized_file2"
 
+	# Modifying the ALTER error msg changes for different yb versions in the failed.sql to match expected failed.sql
+ 	sed -i -E 's#ALTER action CLUSTER ON#ALTER TABLE CLUSTER#g' "$normalized_file1"
+	sed -i -E 's#ALTER action DISABLE RULE#ALTER TABLE DISABLE RULE#g' "$normalized_file1"
+	sed -i -E 's#ALTER action ALTER COLUMN ... SET#ALTER TABLE ALTER COLUMN#g' "$normalized_file1"
+
     # Compare the normalized files
     compare_files "$normalized_file1" "$normalized_file2"
 

diff --git a/migtests/scripts/resumption.sh b/migtests/scripts/resumption.sh
@@ -93,6 +93,15 @@ main() {
 
 	${SCRIPTS}/resumption.py config.yaml
 
+	step "Run import-data-status"
+	import_data_status
+
+	expected_file="${TEST_DIR}/import_data_status-final-report.json"
+	actual_file="${EXPORT_DIR}/reports/import-data-status-report.json"
+
+	step "Verify import-data-status report"
+	verify_report ${expected_file} ${actual_file}
+
 	step "Clean up"
 	rm -rf "${EXPORT_DIR}"
 	if [ -f "${TEST_DIR}/generate_config.py" ]; then

diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql
@@ -466,3 +466,7 @@ CREATE TABLE users_unique_nulls_not_distinct_index (
     id INTEGER PRIMARY KEY,
     email TEXT
 );
+
+CREATE TABLE tbl_comp (id int, val text COMPRESSION pglz, val1 text);
+
+ALTER TABLE tbl_comp ALTER COLUMN val1 SET COMPRESSION lz4;