Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: added support columns collection #262

Merged
merged 1 commit into from
Feb 2, 2025
Merged

feat: added support columns collection #262

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
* Fixed: YdbDataReader.GetDataTypeName for optional values.
* Added support for "Columns" collectionName in YdbConnection.GetSchema(Async).

## v0.12.0
* GetUint64(int ordinal) returns a ulong for Uint8, Uint16, Uint32, Uint64 YDB types.
* GetInt64(int ordinal) returns a int for Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32 YDB types.
Expand Down
2 changes: 1 addition & 1 deletion src/Ydb.Sdk/src/Ado/YdbDataReader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ private static void CheckOffsets<T>(long dataOffset, T[]? buffer, int bufferOffs

public override string GetDataTypeName(int ordinal)
{
return ReaderMetadata.GetColumn(ordinal).Type.TypeId.ToString();
return ReaderMetadata.GetColumn(ordinal).Type.YqlTableType();
}

public override DateTime GetDateTime(int ordinal)
Expand Down
149 changes: 123 additions & 26 deletions src/Ydb.Sdk/src/Ado/YdbSchema.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
using System.Collections.Immutable;
using System.Data;
using System.Data.Common;
using System.Globalization;
using Ydb.Scheme;
using Ydb.Scheme.V1;
using Ydb.Sdk.Services.Table;
Expand Down Expand Up @@ -28,6 +30,7 @@ public static Task<DataTable> GetSchemaAsync(

// Ydb specific Schema Collections
"TABLES" => GetTables(ydbConnection, restrictions, cancellationToken),
"COLUMNS" => GetColumns(ydbConnection, restrictions, cancellationToken),
"TABLESWITHSTATS" => GetTablesWithStats(ydbConnection, restrictions, cancellationToken),

_ => throw new ArgumentOutOfRangeException(nameof(collectionName), collectionName,
Expand All @@ -40,18 +43,24 @@ private static async Task<DataTable> GetTables(
string?[] restrictions,
CancellationToken cancellationToken)
{
var table = new DataTable("Tables");
table.Columns.Add("table_name", typeof(string));
table.Columns.Add("table_type", typeof(string));
var table = new DataTable("Tables")
{
Locale = CultureInfo.InvariantCulture,
Columns =
{
new DataColumn("table_name"),
new DataColumn("table_type")
}
};

var tableName = restrictions[0];
var tableType = restrictions[1];
var database = ydbConnection.Database;

if (tableName == null) // tableName isn't set
{
foreach (var tupleTable in await ListTables(ydbConnection.Session.Driver,
WithSuffix(database), database, tableType, cancellationToken))
foreach (var tupleTable in
await ListTables(ydbConnection, WithSuffix(database), database, tableType, cancellationToken))
{
table.Rows.Add(tupleTable.TableName, tupleTable.TableType);
}
Expand All @@ -61,7 +70,6 @@ private static async Task<DataTable> GetTables(
await AppendDescribeTable(
ydbConnection: ydbConnection,
describeTableSettings: new DescribeTableSettings { CancellationToken = cancellationToken },
database: database,
tableName: tableName,
tableType: tableType,
(_, type) => { table.Rows.Add(tableName, type); });
Expand All @@ -75,27 +83,32 @@ private static async Task<DataTable> GetTablesWithStats(
string?[] restrictions,
CancellationToken cancellationToken)
{
var table = new DataTable("TablesWithStats");
table.Columns.Add("table_name", typeof(string));
table.Columns.Add("table_type", typeof(string));
table.Columns.Add("rows_estimate", typeof(ulong));
table.Columns.Add("creation_time", typeof(DateTime));
table.Columns.Add("modification_time", typeof(DateTime));
var table = new DataTable("TablesWithStats")
{
Locale = CultureInfo.InvariantCulture,
Columns =
{
new DataColumn("table_name"),
new DataColumn("table_type"),
new DataColumn("rows_estimate", typeof(ulong)),
new DataColumn("creation_time", typeof(DateTime)),
new DataColumn("modification_time", typeof(DateTime))
}
};

var tableName = restrictions[0];
var tableType = restrictions[1];
var database = ydbConnection.Database;

if (tableName == null) // tableName isn't set
{
foreach (var tupleTable in await ListTables(ydbConnection.Session.Driver,
WithSuffix(database), database, tableType, cancellationToken))
foreach (var tupleTable in
await ListTables(ydbConnection, WithSuffix(database), database, tableType, cancellationToken))
{
await AppendDescribeTable(
ydbConnection: ydbConnection,
describeTableSettings: new DescribeTableSettings { CancellationToken = cancellationToken }
.WithTableStats(),
database: database,
tableName: tupleTable.TableName,
tableType: tableType,
(describeTableResult, type) =>
Expand All @@ -117,7 +130,6 @@ await AppendDescribeTable(
ydbConnection: ydbConnection,
describeTableSettings: new DescribeTableSettings { CancellationToken = cancellationToken }
.WithTableStats(),
database: database,
tableName: tableName,
tableType: tableType,
(describeTableResult, type) =>
Expand All @@ -136,18 +148,74 @@ await AppendDescribeTable(
return table;
}

private static async Task<DataTable> GetColumns(
YdbConnection ydbConnection,
string?[] restrictions,
CancellationToken cancellationToken)
{
var table = new DataTable("Columns")
{
Locale = CultureInfo.InvariantCulture,
Columns =
{
new DataColumn("table_name"),
new DataColumn("column_name"),
new DataColumn("ordinal_position", typeof(int)),
new DataColumn("is_nullable"),
new DataColumn("data_type"),
new DataColumn("family_name")
}
};
var tableNameRestriction = restrictions[0];
var columnName = restrictions[1];

var tableNames = await ListTableNames(ydbConnection, tableNameRestriction, cancellationToken);
foreach (var tableName in tableNames)
{
await AppendDescribeTable(
ydbConnection,
new DescribeTableSettings { CancellationToken = cancellationToken },
tableName,
null,
(result, _) =>
{
for (var ordinal = 0; ordinal < result.Columns.Count; ordinal++)
{
var column = result.Columns[ordinal];

if (!column.Name.IsPattern(columnName))
{
continue;
}

var row = table.Rows.Add();
var type = column.Type;

row["table_name"] = tableName;
row["column_name"] = column.Name;
row["ordinal_position"] = ordinal;
row["is_nullable"] = type.TypeCase == Type.TypeOneofCase.OptionalType ? "YES" : "NO";
row["data_type"] = type.YqlTableType();
row["family_name"] = column.Family;
}
}
);
}

return table;
}

private static async Task AppendDescribeTable(
YdbConnection ydbConnection,
DescribeTableSettings describeTableSettings,
string database,
string tableName,
string? tableType,
Action<DescribeTableResult, string> appendInTable)
{
try
{
var describeResponse = await ydbConnection.Session
.DescribeTable(WithSuffix(database) + tableName, describeTableSettings);
.DescribeTable(WithSuffix(ydbConnection.Database) + tableName, describeTableSettings);

if (describeResponse.Operation.Status == StatusIds.Types.StatusCode.SchemeError)
{
Expand All @@ -174,7 +242,7 @@ private static async Task AppendDescribeTable(
_ => throw new YdbException($"Unexpected entry type for Table: {describeRes.Self.Type}")
};

if (type.IsTableType(tableType))
if (type.IsPattern(tableType))
{
appendInTable(describeRes, type);
}
Expand All @@ -187,8 +255,26 @@ private static async Task AppendDescribeTable(
}
}

private static async Task<List<(string TableName, string TableType)>> ListTables(
Driver driver,
private static async Task<IReadOnlyCollection<string>> ListTableNames(
YdbConnection ydbConnection,
string? tableName,
CancellationToken cancellationToken)
{
var database = ydbConnection.Database;

return tableName != null
? new List<string> { tableName }
: (await ListTables(
ydbConnection,
WithSuffix(database),
database,
null,
cancellationToken
)).Select(tuple => tuple.TableName).ToImmutableList();
}

private static async Task<IReadOnlyCollection<(string TableName, string TableType)>> ListTables(
YdbConnection ydbConnection,
string databasePath,
string path,
string? tableType,
Expand All @@ -198,7 +284,7 @@ private static async Task AppendDescribeTable(
{
var fullPath = WithSuffix(path);
var tables = new List<(string, string)>();
var response = await driver.UnaryCall(
var response = await ydbConnection.Session.Driver.UnaryCall(
SchemeService.ListDirectoryMethod,
new ListDirectoryRequest { Path = fullPath },
new GrpcRequestSettings { CancellationToken = cancellationToken }
Expand All @@ -220,22 +306,23 @@ private static async Task AppendDescribeTable(
{
case Entry.Types.Type.Table:
var type = tablePath.IsSystem() ? "SYSTEM_TABLE" : "TABLE";
if (type.IsTableType(tableType))
if (type.IsPattern(tableType))
{
tables.Add((tablePath, type));
}

break;
case Entry.Types.Type.ColumnTable:
if ("COLUMN_TABLE".IsTableType(tableType))
if ("COLUMN_TABLE".IsPattern(tableType))
{
tables.Add((tablePath, "COLUMN_TABLE"));
}

break;
case Entry.Types.Type.Directory:
tables.AddRange(
await ListTables(driver, databasePath, fullPath + entry.Name, tableType, cancellationToken)
await ListTables(ydbConnection, databasePath, fullPath + entry.Name, tableType,
cancellationToken)
);
break;
case Entry.Types.Type.Unspecified:
Expand Down Expand Up @@ -333,6 +420,7 @@ private static DataTable GetMetaDataCollections()
// Ydb Specific Schema Collections
table.Rows.Add("Tables", 2, 1);
table.Rows.Add("TablesWithStats", 2, 1);
table.Rows.Add("Columns", 2, 2);

return table;
}
Expand All @@ -350,6 +438,8 @@ private static DataTable GetRestrictions()
table.Rows.Add("Tables", "TableType", "TABLE_TYPE", 2);
table.Rows.Add("TablesWithStats", "Table", "TABLE_NAME", 1);
table.Rows.Add("TablesWithStats", "TableType", "TABLE_TYPE", 2);
table.Rows.Add("Columns", "Table", "TABLE_NAME", 1);
table.Rows.Add("Columns", "Column", "COLUMN_NAME", 2);

return table;
}
Expand All @@ -366,8 +456,15 @@ private static bool IsSystem(this string tablePath)
|| tablePath.StartsWith(".sys_health_dev/");
}

private static bool IsTableType(this string tableType, string? expectedTableType)
private static bool IsPattern(this string tableType, string? expectedTableType)
{
return expectedTableType == null || expectedTableType.Equals(tableType, StringComparison.OrdinalIgnoreCase);
}

internal static string YqlTableType(this Type type)
{
return type.TypeCase == Type.TypeOneofCase.OptionalType
? type.OptionalType.Item.TypeId.ToString()
: type.TypeId.ToString();
}
}
10 changes: 10 additions & 0 deletions src/Ydb.Sdk/tests/Ado/YdbCommandTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,9 @@ public async Task ExecuteDbDataReader_WhenSelectManyResultSet_ReturnYdbDataReade
Assert.True(ydbDataReader.HasRows);
// Read 2 result set
Assert.True(await ydbDataReader.NextResultAsync());
Assert.Equal("Bool", ydbDataReader.GetDataTypeName(0));
Assert.Equal("Double", ydbDataReader.GetDataTypeName(1));
Assert.Equal("Int32", ydbDataReader.GetDataTypeName(2));
for (var i = 0; i < 1500; i++)
{
// Read meta info
Expand All @@ -214,12 +217,19 @@ public async Task ExecuteDbDataReader_WhenSelectManyResultSet_ReturnYdbDataReade

// Read 3 result set
Assert.True(await ydbDataReader.NextResultAsync());
Assert.Equal("Int8", ydbDataReader.GetDataTypeName(0));
Assert.Equal("null_field", ydbDataReader.GetName(0));
Assert.True(await ydbDataReader.ReadAsync());
Assert.True(ydbDataReader.IsDBNull(0));
Assert.Equal(DBNull.Value, ydbDataReader.GetValue(0));
Assert.False(await ydbDataReader.ReadAsync());

// Read 4 result set
Assert.True(await ydbDataReader.NextResultAsync());
Assert.Equal("Datetime", ydbDataReader.GetDataTypeName(0));
Assert.Equal("Key", ydbDataReader.GetName(0));
Assert.Equal("Timestamp", ydbDataReader.GetDataTypeName(1));
Assert.Equal("Value", ydbDataReader.GetName(1));
Assert.True(await ydbDataReader.ReadAsync());
Assert.Equal(dateTime, ydbDataReader.GetDateTime(0));
Assert.Equal(timestamp, ydbDataReader.GetDateTime(1));
Expand Down
Loading
Loading