diff --git a/go/base/context.go b/go/base/context.go index f8054d6dc..6f243c7aa 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -213,6 +213,7 @@ type MigrationContext struct { GhostTableUniqueKeys [](*sql.UniqueKey) UniqueKey *sql.UniqueKey SharedColumns *sql.ColumnList + SharedVirtualColumns *sql.ColumnList ColumnRenameMap map[string]string DroppedColumnsMap map[string]bool MappedSharedColumns *sql.ColumnList diff --git a/go/logic/applier.go b/go/logic/applier.go index a7a845407..a500def56 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -1001,7 +1001,7 @@ func (this *Applier) buildDMLEventQuery(dmlEvent *binlog.BinlogDMLEvent) (result results = append(results, this.buildDMLEventQuery(dmlEvent)...) return results } - query, sharedArgs, uniqueKeyArgs, err := sql.BuildDMLUpdateQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues()) + query, sharedArgs, uniqueKeyArgs, err := sql.BuildDMLUpdateQuery(dmlEvent.DatabaseName, this.migrationContext.GetGhostTableName(), this.migrationContext.OriginalTableColumns, this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, this.migrationContext.SharedVirtualColumns, &this.migrationContext.UniqueKey.Columns, dmlEvent.NewColumnValues.AbstractValues(), dmlEvent.WhereColumnValues.AbstractValues()) args := sqlutils.Args() args = append(args, sharedArgs...) args = append(args, uniqueKeyArgs...) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 0e0c2a321..2766ad8ae 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -171,7 +171,7 @@ func (this *Inspector) inspectOriginalAndGhostTables() (err error) { } } - this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap) + this.migrationContext.SharedColumns, this.migrationContext.MappedSharedColumns, this.migrationContext.SharedVirtualColumns = this.getSharedColumns(this.migrationContext.OriginalTableColumns, this.migrationContext.GhostTableColumns, this.migrationContext.OriginalTableVirtualColumns, this.migrationContext.GhostTableVirtualColumns, this.migrationContext.ColumnRenameMap) this.migrationContext.Log.Infof("Shared columns are %s", this.migrationContext.SharedColumns) // By fact that a non-empty unique key exists we also know the shared columns are non-empty @@ -720,8 +720,9 @@ func (this *Inspector) getSharedUniqueKeys(originalUniqueKeys, ghostUniqueKeys [ } // getSharedColumns returns the intersection of two lists of columns in same order as the first list -func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, originalVirtualColumns, ghostVirtualColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList) { +func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.ColumnList, originalVirtualColumns, ghostVirtualColumns *sql.ColumnList, columnRenameMap map[string]string) (*sql.ColumnList, *sql.ColumnList, *sql.ColumnList) { sharedColumnNames := []string{} + sharedVirtualColumnNames := []string{} for _, originalColumn := range originalColumns.Names() { isSharedColumn := false for _, ghostColumn := range ghostColumns.Names() { @@ -754,6 +755,29 @@ func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.Colum sharedColumnNames = append(sharedColumnNames, originalColumn) } } + // Virtual columns + for _, originalColumn := range originalVirtualColumns.Names() { + isSharedColumn := false + for _, ghostColumn := range ghostVirtualColumns.Names() { + if strings.EqualFold(originalColumn, ghostColumn) { + isSharedColumn = true + break + } + if strings.EqualFold(columnRenameMap[originalColumn], ghostColumn) { + isSharedColumn = true + break + } + } + for droppedColumn := range this.migrationContext.DroppedColumnsMap { + if strings.EqualFold(originalColumn, droppedColumn) { + isSharedColumn = false + break + } + } + if isSharedColumn { + sharedVirtualColumnNames = append(sharedVirtualColumnNames, originalColumn) + } + } mappedSharedColumnNames := []string{} for _, columnName := range sharedColumnNames { if mapped, ok := columnRenameMap[columnName]; ok { @@ -762,7 +786,7 @@ func (this *Inspector) getSharedColumns(originalColumns, ghostColumns *sql.Colum mappedSharedColumnNames = append(mappedSharedColumnNames, columnName) } } - return sql.NewColumnList(sharedColumnNames), sql.NewColumnList(mappedSharedColumnNames) + return sql.NewColumnList(sharedColumnNames), sql.NewColumnList(mappedSharedColumnNames), sql.NewColumnList(sharedVirtualColumnNames) } // showCreateTable returns the `show create table` statement for given table diff --git a/go/logic/migrator.go b/go/logic/migrator.go index c12c21fc3..876ee0f8f 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -796,6 +796,9 @@ func (this *Migrator) initiateInspector() (err error) { } else if this.migrationContext.InspectorIsAlsoApplier() && !this.migrationContext.AllowedRunningOnMaster { return fmt.Errorf("It seems like this migration attempt to run directly on master. Preferably it would be executed on a replica (and this reduces load from the master). To proceed please provide --allow-on-master. Inspector config=%+v, applier config=%+v", this.migrationContext.InspectorConnectionConfig, this.migrationContext.ApplierConnectionConfig) } + if this.migrationContext.InspectorIsAlsoApplier() && this.migrationContext.SwitchToRowBinlogFormat { + return fmt.Errorf("--switch-to-rbr is only allowed when inspecting a replica. This migration seems to run on the primary %+v", *this.migrationContext.ApplierConnectionConfig.ImpliedKey) + } if err := this.inspector.validateLogSlaveUpdates(); err != nil { return err } diff --git a/go/sql/builder.go b/go/sql/builder.go index 7fe366c6f..c37de5523 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -461,7 +461,9 @@ func BuildDMLInsertQuery(databaseName, tableName string, tableColumns, sharedCol return result, sharedArgs, nil } -func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedColumns, mappedSharedColumns, uniqueKeyColumns *ColumnList, valueArgs, whereArgs []interface{}) (result string, sharedArgs, uniqueKeyArgs []interface{}, err error) { +func BuildDMLUpdateQuery(databaseName, tableName string, + tableColumns, sharedColumns, mappedSharedColumns, sharedVirtualColumns, uniqueKeyColumns *ColumnList, + valueArgs, whereArgs []interface{}) (result string, sharedArgs, uniqueKeyArgs []interface{}, err error) { if len(valueArgs) != tableColumns.Len() { return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("value args count differs from table column count in BuildDMLUpdateQuery") } @@ -471,7 +473,7 @@ func BuildDMLUpdateQuery(databaseName, tableName string, tableColumns, sharedCol if !sharedColumns.IsSubsetOf(tableColumns) { return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("shared columns is not a subset of table columns in BuildDMLUpdateQuery") } - if !uniqueKeyColumns.IsSubsetOf(sharedColumns) { + if !uniqueKeyColumns.IsSubsetOf(ConcatenateColumnList(sharedColumns, sharedVirtualColumns)) { return result, sharedArgs, uniqueKeyArgs, fmt.Errorf("unique key columns is not a subset of shared columns in BuildDMLUpdateQuery") } if sharedColumns.Len() == 0 { diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index 299824269..2c700082f 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -544,7 +544,8 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"position"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + generatedColumns := NewColumnList([]string{}) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, generatedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ @@ -560,7 +561,8 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"position", "name"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + generatedColumns := NewColumnList([]string{}) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, generatedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ @@ -576,7 +578,8 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"age"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + generatedColumns := NewColumnList([]string{}) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, generatedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ @@ -592,7 +595,8 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"age", "position", "id", "name"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + generatedColumns := NewColumnList([]string{}) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, generatedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ @@ -608,20 +612,23 @@ func TestBuildDMLUpdateQuery(t *testing.T) { { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{"age", "surprise"}) - _, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + generatedColumns := NewColumnList([]string{}) + _, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, generatedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNotNil(err) } { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) uniqueKeyColumns := NewColumnList([]string{}) - _, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + generatedColumns := NewColumnList([]string{}) + _, _, _, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, generatedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNotNil(err) } { sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) mappedColumns := NewColumnList([]string{"id", "name", "role", "age"}) uniqueKeyColumns := NewColumnList([]string{"id"}) - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, mappedColumns, uniqueKeyColumns, valueArgs, whereArgs) + generatedColumns := NewColumnList([]string{}) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, mappedColumns, generatedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ @@ -643,10 +650,11 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) { valueArgs := []interface{}{3, "testname", "newval", int8(-17), int8(-2)} whereArgs := []interface{}{3, "testname", "findme", int8(-3), 56} sharedColumns := NewColumnList([]string{"id", "name", "position", "age"}) + generatedColumns := NewColumnList([]string{}) uniqueKeyColumns := NewColumnList([]string{"position"}) { // test signed - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, generatedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ @@ -663,7 +671,7 @@ func TestBuildDMLUpdateQuerySignedUnsigned(t *testing.T) { // test unsigned sharedColumns.SetUnsigned("age") uniqueKeyColumns.SetUnsigned("position") - query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, uniqueKeyColumns, valueArgs, whereArgs) + query, sharedArgs, uniqueKeyArgs, err := BuildDMLUpdateQuery(databaseName, tableName, tableColumns, sharedColumns, sharedColumns, generatedColumns, uniqueKeyColumns, valueArgs, whereArgs) test.S(t).ExpectNil(err) expected := ` update /* gh-ost mydb.tbl */ diff --git a/go/sql/types.go b/go/sql/types.go index 3c4ce5e85..72c8c6523 100644 --- a/go/sql/types.go +++ b/go/sql/types.go @@ -234,6 +234,19 @@ func (this *ColumnList) IsSubsetOf(other *ColumnList) bool { return true } +// ConcatenateColumnList concatenates two column lists. It does not check for duplicates. +func ConcatenateColumnList(columns1, columns2 *ColumnList) *ColumnList { + result := &ColumnList{} + for _, col := range columns1.Columns() { + result.columns = append(result.columns, col) + } + for _, col := range columns2.Columns() { + result.columns = append(result.columns, col) + } + result.Ordinals = NewColumnsMap(result.columns) + return result +} + func (this *ColumnList) Len() int { return len(this.columns) } diff --git a/localtests/generated-columns57-unique/create.sql b/localtests/generated-columns57-unique/create.sql index 7a63dd984..988a928af 100644 --- a/localtests/generated-columns57-unique/create.sql +++ b/localtests/generated-columns57-unique/create.sql @@ -27,4 +27,6 @@ begin insert into gh_ost_test (id, jsonobj) values (null, '{"_id":19}'); insert into gh_ost_test (id, jsonobj) values (null, '{"_id":23}'); insert into gh_ost_test (id, jsonobj) values (null, '{"_id":27}'); + set @last_insert_id := last_insert_id(); + update gh_ost_test set jsonobj=JSON_OBJECT('_id', 27, 'name', 'carrot') where id=@last_insert_id and idb=27; end ;;