diff --git a/CHANGELOG.md b/CHANGELOG.md index e9280256..11a46fc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +* Fixed: YdbDataReader.GetDataTypeName for optional values. +* Added support for "Columns" collectionName in YdbConnection.GetSchema(Async). + ## v0.12.0 * GetUint64(int ordinal) returns a ulong for Uint8, Uint16, Uint32, Uint64 YDB types. * GetInt64(int ordinal) returns a int for Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32 YDB types. diff --git a/src/Ydb.Sdk/src/Ado/YdbDataReader.cs b/src/Ydb.Sdk/src/Ado/YdbDataReader.cs index 0839ef55..164fb4a7 100644 --- a/src/Ydb.Sdk/src/Ado/YdbDataReader.cs +++ b/src/Ydb.Sdk/src/Ado/YdbDataReader.cs @@ -181,7 +181,7 @@ private static void CheckOffsets(long dataOffset, T[]? buffer, int bufferOffs public override string GetDataTypeName(int ordinal) { - return ReaderMetadata.GetColumn(ordinal).Type.TypeId.ToString(); + return ReaderMetadata.GetColumn(ordinal).Type.YqlTableType(); } public override DateTime GetDateTime(int ordinal) diff --git a/src/Ydb.Sdk/src/Ado/YdbSchema.cs b/src/Ydb.Sdk/src/Ado/YdbSchema.cs index 19061c10..d9ba68c3 100644 --- a/src/Ydb.Sdk/src/Ado/YdbSchema.cs +++ b/src/Ydb.Sdk/src/Ado/YdbSchema.cs @@ -1,5 +1,7 @@ +using System.Collections.Immutable; using System.Data; using System.Data.Common; +using System.Globalization; using Ydb.Scheme; using Ydb.Scheme.V1; using Ydb.Sdk.Services.Table; @@ -28,6 +30,7 @@ public static Task GetSchemaAsync( // Ydb specific Schema Collections "TABLES" => GetTables(ydbConnection, restrictions, cancellationToken), + "COLUMNS" => GetColumns(ydbConnection, restrictions, cancellationToken), "TABLESWITHSTATS" => GetTablesWithStats(ydbConnection, restrictions, cancellationToken), _ => throw new ArgumentOutOfRangeException(nameof(collectionName), collectionName, @@ -40,9 +43,15 @@ private static async Task GetTables( string?[] restrictions, CancellationToken cancellationToken) { - var table = new DataTable("Tables"); - table.Columns.Add("table_name", typeof(string)); - table.Columns.Add("table_type", typeof(string)); + var table = new DataTable("Tables") + { + Locale = CultureInfo.InvariantCulture, + Columns = + { + new DataColumn("table_name"), + new DataColumn("table_type") + } + }; var tableName = restrictions[0]; var tableType = restrictions[1]; @@ -50,8 +59,8 @@ private static async Task GetTables( if (tableName == null) // tableName isn't set { - foreach (var tupleTable in await ListTables(ydbConnection.Session.Driver, - WithSuffix(database), database, tableType, cancellationToken)) + foreach (var tupleTable in + await ListTables(ydbConnection, WithSuffix(database), database, tableType, cancellationToken)) { table.Rows.Add(tupleTable.TableName, tupleTable.TableType); } @@ -61,7 +70,6 @@ private static async Task GetTables( await AppendDescribeTable( ydbConnection: ydbConnection, describeTableSettings: new DescribeTableSettings { CancellationToken = cancellationToken }, - database: database, tableName: tableName, tableType: tableType, (_, type) => { table.Rows.Add(tableName, type); }); @@ -75,12 +83,18 @@ private static async Task GetTablesWithStats( string?[] restrictions, CancellationToken cancellationToken) { - var table = new DataTable("TablesWithStats"); - table.Columns.Add("table_name", typeof(string)); - table.Columns.Add("table_type", typeof(string)); - table.Columns.Add("rows_estimate", typeof(ulong)); - table.Columns.Add("creation_time", typeof(DateTime)); - table.Columns.Add("modification_time", typeof(DateTime)); + var table = new DataTable("TablesWithStats") + { + Locale = CultureInfo.InvariantCulture, + Columns = + { + new DataColumn("table_name"), + new DataColumn("table_type"), + new DataColumn("rows_estimate", typeof(ulong)), + new DataColumn("creation_time", typeof(DateTime)), + new DataColumn("modification_time", typeof(DateTime)) + } + }; var tableName = restrictions[0]; var tableType = restrictions[1]; @@ -88,14 +102,13 @@ private static async Task GetTablesWithStats( if (tableName == null) // tableName isn't set { - foreach (var tupleTable in await ListTables(ydbConnection.Session.Driver, - WithSuffix(database), database, tableType, cancellationToken)) + foreach (var tupleTable in + await ListTables(ydbConnection, WithSuffix(database), database, tableType, cancellationToken)) { await AppendDescribeTable( ydbConnection: ydbConnection, describeTableSettings: new DescribeTableSettings { CancellationToken = cancellationToken } .WithTableStats(), - database: database, tableName: tupleTable.TableName, tableType: tableType, (describeTableResult, type) => @@ -117,7 +130,6 @@ await AppendDescribeTable( ydbConnection: ydbConnection, describeTableSettings: new DescribeTableSettings { CancellationToken = cancellationToken } .WithTableStats(), - database: database, tableName: tableName, tableType: tableType, (describeTableResult, type) => @@ -136,10 +148,66 @@ await AppendDescribeTable( return table; } + private static async Task GetColumns( + YdbConnection ydbConnection, + string?[] restrictions, + CancellationToken cancellationToken) + { + var table = new DataTable("Columns") + { + Locale = CultureInfo.InvariantCulture, + Columns = + { + new DataColumn("table_name"), + new DataColumn("column_name"), + new DataColumn("ordinal_position", typeof(int)), + new DataColumn("is_nullable"), + new DataColumn("data_type"), + new DataColumn("family_name") + } + }; + var tableNameRestriction = restrictions[0]; + var columnName = restrictions[1]; + + var tableNames = await ListTableNames(ydbConnection, tableNameRestriction, cancellationToken); + foreach (var tableName in tableNames) + { + await AppendDescribeTable( + ydbConnection, + new DescribeTableSettings { CancellationToken = cancellationToken }, + tableName, + null, + (result, _) => + { + for (var ordinal = 0; ordinal < result.Columns.Count; ordinal++) + { + var column = result.Columns[ordinal]; + + if (!column.Name.IsPattern(columnName)) + { + continue; + } + + var row = table.Rows.Add(); + var type = column.Type; + + row["table_name"] = tableName; + row["column_name"] = column.Name; + row["ordinal_position"] = ordinal; + row["is_nullable"] = type.TypeCase == Type.TypeOneofCase.OptionalType ? "YES" : "NO"; + row["data_type"] = type.YqlTableType(); + row["family_name"] = column.Family; + } + } + ); + } + + return table; + } + private static async Task AppendDescribeTable( YdbConnection ydbConnection, DescribeTableSettings describeTableSettings, - string database, string tableName, string? tableType, Action appendInTable) @@ -147,7 +215,7 @@ private static async Task AppendDescribeTable( try { var describeResponse = await ydbConnection.Session - .DescribeTable(WithSuffix(database) + tableName, describeTableSettings); + .DescribeTable(WithSuffix(ydbConnection.Database) + tableName, describeTableSettings); if (describeResponse.Operation.Status == StatusIds.Types.StatusCode.SchemeError) { @@ -174,7 +242,7 @@ private static async Task AppendDescribeTable( _ => throw new YdbException($"Unexpected entry type for Table: {describeRes.Self.Type}") }; - if (type.IsTableType(tableType)) + if (type.IsPattern(tableType)) { appendInTable(describeRes, type); } @@ -187,8 +255,26 @@ private static async Task AppendDescribeTable( } } - private static async Task> ListTables( - Driver driver, + private static async Task> ListTableNames( + YdbConnection ydbConnection, + string? tableName, + CancellationToken cancellationToken) + { + var database = ydbConnection.Database; + + return tableName != null + ? new List { tableName } + : (await ListTables( + ydbConnection, + WithSuffix(database), + database, + null, + cancellationToken + )).Select(tuple => tuple.TableName).ToImmutableList(); + } + + private static async Task> ListTables( + YdbConnection ydbConnection, string databasePath, string path, string? tableType, @@ -198,7 +284,7 @@ private static async Task AppendDescribeTable( { var fullPath = WithSuffix(path); var tables = new List<(string, string)>(); - var response = await driver.UnaryCall( + var response = await ydbConnection.Session.Driver.UnaryCall( SchemeService.ListDirectoryMethod, new ListDirectoryRequest { Path = fullPath }, new GrpcRequestSettings { CancellationToken = cancellationToken } @@ -220,14 +306,14 @@ private static async Task AppendDescribeTable( { case Entry.Types.Type.Table: var type = tablePath.IsSystem() ? "SYSTEM_TABLE" : "TABLE"; - if (type.IsTableType(tableType)) + if (type.IsPattern(tableType)) { tables.Add((tablePath, type)); } break; case Entry.Types.Type.ColumnTable: - if ("COLUMN_TABLE".IsTableType(tableType)) + if ("COLUMN_TABLE".IsPattern(tableType)) { tables.Add((tablePath, "COLUMN_TABLE")); } @@ -235,7 +321,8 @@ private static async Task AppendDescribeTable( break; case Entry.Types.Type.Directory: tables.AddRange( - await ListTables(driver, databasePath, fullPath + entry.Name, tableType, cancellationToken) + await ListTables(ydbConnection, databasePath, fullPath + entry.Name, tableType, + cancellationToken) ); break; case Entry.Types.Type.Unspecified: @@ -333,6 +420,7 @@ private static DataTable GetMetaDataCollections() // Ydb Specific Schema Collections table.Rows.Add("Tables", 2, 1); table.Rows.Add("TablesWithStats", 2, 1); + table.Rows.Add("Columns", 2, 2); return table; } @@ -350,6 +438,8 @@ private static DataTable GetRestrictions() table.Rows.Add("Tables", "TableType", "TABLE_TYPE", 2); table.Rows.Add("TablesWithStats", "Table", "TABLE_NAME", 1); table.Rows.Add("TablesWithStats", "TableType", "TABLE_TYPE", 2); + table.Rows.Add("Columns", "Table", "TABLE_NAME", 1); + table.Rows.Add("Columns", "Column", "COLUMN_NAME", 2); return table; } @@ -366,8 +456,15 @@ private static bool IsSystem(this string tablePath) || tablePath.StartsWith(".sys_health_dev/"); } - private static bool IsTableType(this string tableType, string? expectedTableType) + private static bool IsPattern(this string tableType, string? expectedTableType) { return expectedTableType == null || expectedTableType.Equals(tableType, StringComparison.OrdinalIgnoreCase); } + + internal static string YqlTableType(this Type type) + { + return type.TypeCase == Type.TypeOneofCase.OptionalType + ? type.OptionalType.Item.TypeId.ToString() + : type.TypeId.ToString(); + } } diff --git a/src/Ydb.Sdk/tests/Ado/YdbCommandTests.cs b/src/Ydb.Sdk/tests/Ado/YdbCommandTests.cs index 061cdb60..98d4ac67 100644 --- a/src/Ydb.Sdk/tests/Ado/YdbCommandTests.cs +++ b/src/Ydb.Sdk/tests/Ado/YdbCommandTests.cs @@ -196,6 +196,9 @@ public async Task ExecuteDbDataReader_WhenSelectManyResultSet_ReturnYdbDataReade Assert.True(ydbDataReader.HasRows); // Read 2 result set Assert.True(await ydbDataReader.NextResultAsync()); + Assert.Equal("Bool", ydbDataReader.GetDataTypeName(0)); + Assert.Equal("Double", ydbDataReader.GetDataTypeName(1)); + Assert.Equal("Int32", ydbDataReader.GetDataTypeName(2)); for (var i = 0; i < 1500; i++) { // Read meta info @@ -214,12 +217,19 @@ public async Task ExecuteDbDataReader_WhenSelectManyResultSet_ReturnYdbDataReade // Read 3 result set Assert.True(await ydbDataReader.NextResultAsync()); + Assert.Equal("Int8", ydbDataReader.GetDataTypeName(0)); + Assert.Equal("null_field", ydbDataReader.GetName(0)); Assert.True(await ydbDataReader.ReadAsync()); Assert.True(ydbDataReader.IsDBNull(0)); + Assert.Equal(DBNull.Value, ydbDataReader.GetValue(0)); Assert.False(await ydbDataReader.ReadAsync()); // Read 4 result set Assert.True(await ydbDataReader.NextResultAsync()); + Assert.Equal("Datetime", ydbDataReader.GetDataTypeName(0)); + Assert.Equal("Key", ydbDataReader.GetName(0)); + Assert.Equal("Timestamp", ydbDataReader.GetDataTypeName(1)); + Assert.Equal("Value", ydbDataReader.GetName(1)); Assert.True(await ydbDataReader.ReadAsync()); Assert.Equal(dateTime, ydbDataReader.GetDateTime(0)); Assert.Equal(timestamp, ydbDataReader.GetDateTime(1)); diff --git a/src/Ydb.Sdk/tests/Ado/YdbSchemaTests.cs b/src/Ydb.Sdk/tests/Ado/YdbSchemaTests.cs index 90c7487f..c8249c11 100644 --- a/src/Ydb.Sdk/tests/Ado/YdbSchemaTests.cs +++ b/src/Ydb.Sdk/tests/Ado/YdbSchemaTests.cs @@ -8,8 +8,17 @@ namespace Ydb.Sdk.Tests.Ado; public class YdbSchemaTests : YdbAdoNetFixture { + private readonly string _table1; + private readonly string _table2; + private readonly string _table3; + private readonly HashSet _tableNames; + public YdbSchemaTests(YdbFactoryFixture fixture) : base(fixture) { + _table1 = $"a/b/{Utils.Net}_{Random.Shared.Next()}"; + _table2 = $"a/{Utils.Net}_{Random.Shared.Next()}"; + _table3 = $"{Utils.Net}_{Random.Shared.Next()}"; + _tableNames = new HashSet { _table1, _table2, _table3 }; } [Fact] @@ -17,51 +26,28 @@ public async Task GetSchema_WhenTablesCollection_ReturnAllTables() { await using var ydbConnection = await CreateOpenConnectionAsync(); - var table1 = $"a/b/{Utils.Net}"; - var table2 = $"a/{Utils.Net}"; - var table3 = $"{Utils.Net}"; - - var tableNames = new HashSet { table1, table2, table3 }; - - await new YdbCommand(ydbConnection) - { - CommandText = $@" -CREATE TABLE `{table1}` (a Int32, b Int32, PRIMARY KEY(a)); -CREATE TABLE `{table2}` (a Int32, b Int32, PRIMARY KEY(a)); -CREATE TABLE `{table3}` (a Int32, b Int32, PRIMARY KEY(a)); -" - }.ExecuteNonQueryAsync(); - var table = await ydbConnection.GetSchemaAsync("Tables", new[] { null, "TABLE" }); foreach (DataRow row in table.Rows) { - tableNames.Remove(row["table_name"].ToString()!); + _tableNames.Remove(row["table_name"].ToString()!); } - Assert.Empty(tableNames); + Assert.Empty(_tableNames); - var singleTable1 = await ydbConnection.GetSchemaAsync("Tables", new[] { table1, "TABLE" }); + var singleTable1 = await ydbConnection.GetSchemaAsync("Tables", new[] { _table1, "TABLE" }); Assert.Equal(1, singleTable1.Rows.Count); - Assert.Equal(table1, singleTable1.Rows[0]["table_name"].ToString()); + Assert.Equal(_table1, singleTable1.Rows[0]["table_name"].ToString()); Assert.Equal("TABLE", singleTable1.Rows[0]["table_type"].ToString()); - var singleTable2 = await ydbConnection.GetSchemaAsync("Tables", new[] { table2, null }); + var singleTable2 = await ydbConnection.GetSchemaAsync("Tables", new[] { _table2, null }); Assert.Equal(1, singleTable2.Rows.Count); - Assert.Equal(table2, singleTable2.Rows[0]["table_name"].ToString()); + Assert.Equal(_table2, singleTable2.Rows[0]["table_name"].ToString()); Assert.Equal("TABLE", singleTable2.Rows[0]["table_type"].ToString()); // not found case var notFound = await ydbConnection.GetSchemaAsync("Tables", new[] { "not_found", null }); Assert.Equal(0, notFound.Rows.Count); - - await new YdbCommand(ydbConnection) - { - CommandText = $@" -DROP TABLE `{table1}`; -DROP TABLE `{table2}`; -DROP TABLE `{table3}`;" - }.ExecuteNonQueryAsync(); } [Fact] @@ -69,45 +55,30 @@ public async Task GetSchema_WhenTablesWithStatsCollection_ReturnAllTables() { await using var ydbConnection = await CreateOpenConnectionAsync(); - var table1 = $"a/b/{Utils.Net}_for_stats"; - var table2 = $"a/{Utils.Net}_for_stats"; - var table3 = $"{Utils.Net}_for_stats"; - - var tableNames = new HashSet { table1, table2, table3 }; - - await new YdbCommand(ydbConnection) - { - CommandText = $@" -CREATE TABLE `{table1}` (a Int32, b Int32, PRIMARY KEY(a)); -CREATE TABLE `{table2}` (a Int32, b Int32, PRIMARY KEY(a)); -CREATE TABLE `{table3}` (a Int32, b Int32, PRIMARY KEY(a)); -" - }.ExecuteNonQueryAsync(); - var table = await ydbConnection.GetSchemaAsync("TablesWithStats", new[] { null, "TABLE" }); foreach (DataRow row in table.Rows) { - tableNames.Remove(row["table_name"].ToString()!); + _tableNames.Remove(row["table_name"].ToString()!); Assert.NotNull(row["rows_estimate"]); Assert.NotNull(row["creation_time"]); Assert.NotNull(row["modification_time"]); } - Assert.Empty(tableNames); + Assert.Empty(_tableNames); - var singleTable1 = await ydbConnection.GetSchemaAsync("TablesWithStats", new[] { table1, "TABLE" }); + var singleTable1 = await ydbConnection.GetSchemaAsync("TablesWithStats", new[] { _table1, "TABLE" }); Assert.Equal(1, singleTable1.Rows.Count); - Assert.Equal(table1, singleTable1.Rows[0]["table_name"].ToString()); + Assert.Equal(_table1, singleTable1.Rows[0]["table_name"].ToString()); Assert.Equal("TABLE", singleTable1.Rows[0]["table_type"].ToString()); Assert.NotNull(singleTable1.Rows[0]["rows_estimate"]); Assert.NotNull(singleTable1.Rows[0]["creation_time"]); Assert.NotNull(singleTable1.Rows[0]["modification_time"]); - var singleTable2 = await ydbConnection.GetSchemaAsync("TablesWithStats", new[] { table2, null }); + var singleTable2 = await ydbConnection.GetSchemaAsync("TablesWithStats", new[] { _table2, null }); Assert.Equal(1, singleTable2.Rows.Count); - Assert.Equal(table2, singleTable2.Rows[0]["table_name"].ToString()); + Assert.Equal(_table2, singleTable2.Rows[0]["table_name"].ToString()); Assert.Equal("TABLE", singleTable2.Rows[0]["table_type"].ToString()); Assert.NotNull(singleTable2.Rows[0]["rows_estimate"]); Assert.NotNull(singleTable2.Rows[0]["creation_time"]); @@ -116,13 +87,108 @@ public async Task GetSchema_WhenTablesWithStatsCollection_ReturnAllTables() // not found case var notFound = await ydbConnection.GetSchemaAsync("Tables", new[] { "not_found", null }); Assert.Equal(0, notFound.Rows.Count); + } + + [Fact] + public async Task GetSchema_WhenColumnsCollection_ReturnAllColumns() + { + await using var ydbConnection = await CreateOpenConnectionAsync(); + + foreach (var tableName in _tableNames) + { + var dataTable = await ydbConnection.GetSchemaAsync("Columns", new[] { tableName, null }); + + Assert.Equal(2, dataTable.Rows.Count); + + const int ordinalA = 0; + var columnA = dataTable.Rows[ordinalA]; + Assert.Equal(tableName, columnA["table_name"]); + CheckColumnA(columnA); + + const int ordinalB = 1; + var columnB = dataTable.Rows[ordinalB]; + Assert.Equal(tableName, columnB["table_name"]); + CheckColumnB(columnB); + } + + var rowsA = (await ydbConnection.GetSchemaAsync("Columns", new[] { null, "a" })).Rows; + Assert.Equal(3, rowsA.Count); + for (var i = 0; i < rowsA.Count; i++) + { + CheckColumnA(rowsA[i]); + } + + var rowsB = (await ydbConnection.GetSchemaAsync("Columns", new[] { null, "b" })).Rows; + Assert.Equal(3, rowsB.Count); + for (var i = 0; i < rowsB.Count; i++) + { + CheckColumnB(rowsB[i]); + } + + foreach (var tableName in _tableNames) + { + var dataTable = await ydbConnection.GetSchemaAsync("Columns", new[] { tableName, "a" }); + Assert.Equal(1, dataTable.Rows.Count); + var columnA = dataTable.Rows[0]; + Assert.Equal(tableName, columnA["table_name"]); + CheckColumnA(columnA); + } + + foreach (var tableName in _tableNames) + { + var dataTable = await ydbConnection.GetSchemaAsync("Columns", new[] { tableName, "b" }); + Assert.Equal(1, dataTable.Rows.Count); + var columnB = dataTable.Rows[0]; + Assert.Equal(tableName, columnB["table_name"]); + CheckColumnB(columnB); + } + + return; + + void CheckColumnA(DataRow columnA) + { + Assert.Equal("a", columnA["column_name"]); + Assert.Equal(0, columnA["ordinal_position"]); + Assert.Equal("NO", columnA["is_nullable"]); + Assert.Equal("Int32", columnA["data_type"]); + Assert.Empty((string)columnA["family_name"]); + } + + void CheckColumnB(DataRow columnB) + { + Assert.Equal("b", columnB["column_name"]); + Assert.Equal(1, columnB["ordinal_position"]); + Assert.Equal("YES", columnB["is_nullable"]); + Assert.Equal("Int32", columnB["data_type"]); + Assert.Empty((string)columnB["family_name"]); + } + } + + protected override async Task OnInitializeAsync() + { + await using var ydbConnection = await CreateOpenConnectionAsync(); + + await new YdbCommand(ydbConnection) + { + CommandText = $@" + CREATE TABLE `{_table1}` (a Int32 NOT NULL, b Int32, PRIMARY KEY(a)); + CREATE TABLE `{_table2}` (a Int32 NOT NULL, b Int32, PRIMARY KEY(a)); + CREATE TABLE `{_table3}` (a Int32 NOT NULL, b Int32, PRIMARY KEY(a)); + " + }.ExecuteNonQueryAsync(); + } + + protected override async Task OnDisposeAsync() + { + await using var ydbConnection = await CreateOpenConnectionAsync(); await new YdbCommand(ydbConnection) { CommandText = $@" -DROP TABLE `{table1}`; -DROP TABLE `{table2}`; -DROP TABLE `{table3}`;" + DROP TABLE `{_table1}`; + DROP TABLE `{_table2}`; + DROP TABLE `{_table3}`; + " }.ExecuteNonQueryAsync(); } }