Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move ALTER TYPE TEXT DDL to CREATE for metadata tables of target db #2383

Merged
merged 3 commits into from
Mar 6, 2025
Merged

Move ALTER TYPE TEXT DDL to CREATE for metadata tables of target db #2383

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2e695e7
Moved ALTER to text types DDL on metadata table in CREATE DDL
priyanshi-yb Mar 5, 2025
8a01f1e
Moved ALTER to text types DDL on metadata table in CREATE DDL
priyanshi-yb Mar 5, 2025
8c0ff7d
use quotes around ssl root cert file path in the client wrapper jar c…
priyanshi-yb Mar 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions yb-voyager/src/dbzm/yb_cdc_client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"strings"

log "github.com/sirupsen/logrus"

"github.com/yugabyte/yb-voyager/yb-voyager/src/metadb"
"github.com/yugabyte/yb-voyager/yb-voyager/src/utils"
)
Expand Down Expand Up @@ -81,7 +82,7 @@ func (ybc *YugabyteDBCDCClient) GenerateAndStoreStreamID() (string, error) {
args := fmt.Sprintf("-create -master_addresses %s -table_name %s -db_name %s ", ybc.ybMasterNodes, ybc.tableName, ybc.dbName)

if ybc.sslRootCert != "" {
args += fmt.Sprintf(" -ssl_cert_file %s", ybc.sslRootCert)
args += fmt.Sprintf(" -ssl_cert_file '%s'", ybc.sslRootCert)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what is the context for this change?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I hit this problem when I was trying out with the root cert: If the root cert file name has special characters or spaces, this command will error out if quotes are not present around the file path.
ex. root-cert /<path>/root (2).crt
This might not be really possible in real scenario, but I thought its better to fix it.

}

stdout, err := ybc.runCommand(args)
Expand Down Expand Up @@ -120,7 +121,7 @@ func (ybc *YugabyteDBCDCClient) DeleteStreamID() error {
args := fmt.Sprintf("-delete_stream %s -master_addresses %s ", streamID, ybc.ybMasterNodes)

if ybc.sslRootCert != "" {
args += fmt.Sprintf(" -ssl_cert_file %s", ybc.sslRootCert)
args += fmt.Sprintf(" -ssl_cert_file '%s'", ybc.sslRootCert)
}

_, err = ybc.runCommand(args)
Expand All @@ -145,7 +146,7 @@ func (ybc *YugabyteDBCDCClient) ListMastersNodes() (string, error) {
args := fmt.Sprintf("-list_masters -master_addresses %s -tserver_port %s", ybc.ybServers, tserverPort)

if ybc.sslRootCert != "" {
args += fmt.Sprintf(" -ssl_cert_file %s", ybc.sslRootCert)
args += fmt.Sprintf(" -ssl_cert_file '%s'", ybc.sslRootCert)
}

stdout, err := ybc.runCommand(args)
Expand Down
12 changes: 4 additions & 8 deletions yb-voyager/src/tgtdb/postgres.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,16 +271,13 @@ func (pg *TargetPostgreSQL) CreateVoyagerSchema() error {
fmt.Sprintf(`CREATE SCHEMA IF NOT EXISTS %s;`, BATCH_METADATA_TABLE_SCHEMA),
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
migration_uuid uuid,
data_file_name VARCHAR(250),
data_file_name TEXT,
batch_number INT,
schema_name VARCHAR(250),
table_name VARCHAR(250),
schema_name TEXT,
table_name TEXT,
rows_imported BIGINT,
PRIMARY KEY (migration_uuid, data_file_name, batch_number, schema_name, table_name)
);`, BATCH_METADATA_TABLE_NAME),
fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN data_file_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME),
fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN schema_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME),
fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN table_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME),
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
migration_uuid uuid,
channel_no INT,
Expand All @@ -291,14 +288,13 @@ func (pg *TargetPostgreSQL) CreateVoyagerSchema() error {
PRIMARY KEY (migration_uuid, channel_no));`, EVENT_CHANNELS_METADATA_TABLE_NAME),
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
migration_uuid uuid,
table_name VARCHAR(250),
table_name TEXT,
channel_no INT,
total_events BIGINT,
num_inserts BIGINT,
num_deletes BIGINT,
num_updates BIGINT,
PRIMARY KEY (migration_uuid, table_name, channel_no));`, EVENTS_PER_TABLE_METADATA_TABLE_NAME),
fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN table_name TYPE TEXT;`, EVENTS_PER_TABLE_METADATA_TABLE_NAME),
}

maxAttempts := 12
Expand Down
12 changes: 4 additions & 8 deletions yb-voyager/src/tgtdb/yugabytedb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,16 +347,13 @@ func (yb *TargetYugabyteDB) CreateVoyagerSchema() error {
fmt.Sprintf(`CREATE SCHEMA IF NOT EXISTS %s;`, BATCH_METADATA_TABLE_SCHEMA),
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
migration_uuid uuid,
data_file_name VARCHAR(250),
data_file_name TEXT,
batch_number INT,
schema_name VARCHAR(250),
table_name VARCHAR(250),
schema_name TEXT,
table_name TEXT,
rows_imported BIGINT,
PRIMARY KEY (migration_uuid, data_file_name, batch_number, schema_name, table_name)
);`, BATCH_METADATA_TABLE_NAME),
fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN data_file_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME),
fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN schema_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME),
fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN table_name TYPE TEXT;`, BATCH_METADATA_TABLE_NAME),
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
migration_uuid uuid,
channel_no INT,
Expand All @@ -367,14 +364,13 @@ func (yb *TargetYugabyteDB) CreateVoyagerSchema() error {
PRIMARY KEY (migration_uuid, channel_no));`, EVENT_CHANNELS_METADATA_TABLE_NAME),
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
migration_uuid uuid,
table_name VARCHAR(250),
table_name TEXT,
channel_no INT,
total_events BIGINT,
num_inserts BIGINT,
num_deletes BIGINT,
num_updates BIGINT,
PRIMARY KEY (migration_uuid, table_name, channel_no));`, EVENTS_PER_TABLE_METADATA_TABLE_NAME),
fmt.Sprintf(`ALTER TABLE %s ALTER COLUMN table_name TYPE TEXT;`, EVENTS_PER_TABLE_METADATA_TABLE_NAME),
}

maxAttempts := 12
Expand Down