From 64138aff3ed947b43228c9a124ac3a89995127de Mon Sep 17 00:00:00 2001 From: Liz van Dijk Date: Tue, 17 Oct 2023 12:46:20 +0100 Subject: [PATCH 1/5] Support BIGINT/TINYINT being stored as INTEGER in more situations --- cmd/internal/planetscale_edge_database.go | 5 +++-- cmd/internal/planetscale_edge_database_test.go | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cmd/internal/planetscale_edge_database.go b/cmd/internal/planetscale_edge_database.go index bbf42ed..30c222e 100644 --- a/cmd/internal/planetscale_edge_database.go +++ b/cmd/internal/planetscale_edge_database.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/go-sql-driver/mysql" "github.com/pkg/errors" psdbconnect "github.com/planetscale/airbyte-source/proto/psdbconnect/v1alpha1" "github.com/planetscale/psdb/auth" @@ -137,14 +138,14 @@ func getJsonSchemaType(mysqlType string, treatTinyIntAsBoolean bool) PropertyTyp } if strings.HasPrefix(mysqlType, "bigint") { - return PropertyType{Type: "string", AirbyteType: "big_integer"} + return PropertyType{Type: "integer", AirbyteType: "big_integer"} } if strings.HasPrefix(mysqlType, "datetime") { return PropertyType{Type: "string", CustomFormat: "date-time", AirbyteType: "timestamp_without_timezone"} } - if mysqlType == "tinyint(1)" { + if strings.HasPrefix(mysqlType, "tinyint(1)") { if treatTinyIntAsBoolean { return PropertyType{Type: "boolean"} } diff --git a/cmd/internal/planetscale_edge_database_test.go b/cmd/internal/planetscale_edge_database_test.go index 3637b3a..a952934 100644 --- a/cmd/internal/planetscale_edge_database_test.go +++ b/cmd/internal/planetscale_edge_database_test.go @@ -219,25 +219,37 @@ func TestDiscover_CanPickRightAirbyteType(t *testing.T) { AirbyteType: "", TreatTinyIntAsBoolean: true, }, + { + MysqlType: "tinyint(1) unsigned", + JSONSchemaType: "boolean", + AirbyteType: "", + TreatTinyIntAsBoolean: true, + }, { MysqlType: "tinyint(1)", JSONSchemaType: "integer", AirbyteType: "", TreatTinyIntAsBoolean: false, }, + { + MysqlType: "tinyint(1) unsigned", + JSONSchemaType: "integer", + AirbyteType: "", + TreatTinyIntAsBoolean: false, + }, { MysqlType: "bigint(16)", - JSONSchemaType: "string", + JSONSchemaType: "integer", AirbyteType: "big_integer", }, { MysqlType: "bigint unsigned", - JSONSchemaType: "string", + JSONSchemaType: "integer", AirbyteType: "big_integer", }, { MysqlType: "bigint zerofill", - JSONSchemaType: "string", + JSONSchemaType: "integer", AirbyteType: "big_integer", }, { From 7871fc4754cbc9828b2025a9ddab4018a77e8067 Mon Sep 17 00:00:00 2001 From: Liz van Dijk Date: Tue, 17 Oct 2023 13:23:55 +0100 Subject: [PATCH 2/5] Actually, the problem is not fully on our end, it's Airbyte's default type conversion on the Destination connector for BigQuery. Rolled back bigint change, but kept tinyint adjustment. --- cmd/internal/planetscale_edge_database.go | 2 +- cmd/internal/planetscale_edge_database_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/internal/planetscale_edge_database.go b/cmd/internal/planetscale_edge_database.go index 30c222e..90b8f39 100644 --- a/cmd/internal/planetscale_edge_database.go +++ b/cmd/internal/planetscale_edge_database.go @@ -138,7 +138,7 @@ func getJsonSchemaType(mysqlType string, treatTinyIntAsBoolean bool) PropertyTyp } if strings.HasPrefix(mysqlType, "bigint") { - return PropertyType{Type: "integer", AirbyteType: "big_integer"} + return PropertyType{Type: "string", AirbyteType: "big_integer"} } if strings.HasPrefix(mysqlType, "datetime") { diff --git a/cmd/internal/planetscale_edge_database_test.go b/cmd/internal/planetscale_edge_database_test.go index a952934..f6107e3 100644 --- a/cmd/internal/planetscale_edge_database_test.go +++ b/cmd/internal/planetscale_edge_database_test.go @@ -239,17 +239,17 @@ func TestDiscover_CanPickRightAirbyteType(t *testing.T) { }, { MysqlType: "bigint(16)", - JSONSchemaType: "integer", + JSONSchemaType: "string", AirbyteType: "big_integer", }, { MysqlType: "bigint unsigned", - JSONSchemaType: "integer", + JSONSchemaType: "string", AirbyteType: "big_integer", }, { MysqlType: "bigint zerofill", - JSONSchemaType: "integer", + JSONSchemaType: "string", AirbyteType: "big_integer", }, { From 307616d6cdbe685538e23d2a33db1e4f26164546 Mon Sep 17 00:00:00 2001 From: Liz van Dijk Date: Tue, 17 Oct 2023 13:37:05 +0100 Subject: [PATCH 3/5] removed random import --- cmd/internal/planetscale_edge_database.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/internal/planetscale_edge_database.go b/cmd/internal/planetscale_edge_database.go index 90b8f39..7a3fae2 100644 --- a/cmd/internal/planetscale_edge_database.go +++ b/cmd/internal/planetscale_edge_database.go @@ -7,8 +7,6 @@ import ( "net/http" "strings" "time" - - "github.com/go-sql-driver/mysql" "github.com/pkg/errors" psdbconnect "github.com/planetscale/airbyte-source/proto/psdbconnect/v1alpha1" "github.com/planetscale/psdb/auth" From 0198b3df90fac6a601187648dbe1c1d134420c86 Mon Sep 17 00:00:00 2001 From: Liz van Dijk Date: Tue, 17 Oct 2023 13:40:02 +0100 Subject: [PATCH 4/5] add back line ending per @dbussink --- cmd/internal/planetscale_edge_database.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/internal/planetscale_edge_database.go b/cmd/internal/planetscale_edge_database.go index 7a3fae2..c1aeb51 100644 --- a/cmd/internal/planetscale_edge_database.go +++ b/cmd/internal/planetscale_edge_database.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" "time" + "github.com/pkg/errors" psdbconnect "github.com/planetscale/airbyte-source/proto/psdbconnect/v1alpha1" "github.com/planetscale/psdb/auth" From 004b34aaa13530b06cce50135cc0095f706fd66c Mon Sep 17 00:00:00 2001 From: Liz van Dijk Date: Tue, 17 Oct 2023 13:46:13 +0100 Subject: [PATCH 5/5] removing tab x) --- cmd/internal/planetscale_edge_database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/internal/planetscale_edge_database.go b/cmd/internal/planetscale_edge_database.go index c1aeb51..bd8ecf2 100644 --- a/cmd/internal/planetscale_edge_database.go +++ b/cmd/internal/planetscale_edge_database.go @@ -7,7 +7,7 @@ import ( "net/http" "strings" "time" - + "github.com/pkg/errors" psdbconnect "github.com/planetscale/airbyte-source/proto/psdbconnect/v1alpha1" "github.com/planetscale/psdb/auth"