Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(FIR-38126): struct type #483

Merged
merged 9 commits into from
Dec 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,78 @@ void shouldInsertAndSelectGeography() throws SQLException {
}
}

@Test
@Tag("v2")
void shouldInsertAndSelectStruct() throws SQLException {
Car car1 = Car.builder().make("Ford").sales(12345).ts(new Timestamp(2)).d(new Date(3)).build();

executeStatementFromFile("/statements/prepared-statement/ddl.sql");
try (Connection connection = createConnection()) {

try (PreparedStatement statement = connection
.prepareStatement("INSERT INTO prepared_statement_test (sales, make, ts, d) VALUES (?,?,?,?)")) {
statement.setLong(1, car1.getSales());
statement.setString(2, car1.getMake());
statement.setTimestamp(3, car1.getTs());
statement.setDate(4, car1.getD());
statement.executeUpdate();
}
setParam(connection, "advanced_mode", "true");
setParam(connection, "enable_row_selection", "true");
try (Statement statement = connection.createStatement();
ResultSet rs = statement
.executeQuery("SELECT prepared_statement_test FROM prepared_statement_test")) {
rs.next();
assertEquals(FireboltDataType.STRUCT.name().toLowerCase()
+ "(make text, sales long, ts timestamp null, d date null, signature bytea null, url text null)",
rs.getMetaData().getColumnTypeName(1).toLowerCase());
String expectedJson = String.format("{\"make\":\"%s\",\"sales\":\"%d\",\"ts\":\"%s\",\"d\":\"%s\",\"signature\":null,\"url\":null}",
car1.getMake(), car1.getSales(), car1.getTs().toString(), car1.getD().toString());
assertEquals(expectedJson, rs.getString(1));
}
} finally {
executeStatementFromFile("/statements/prepared-statement/cleanup.sql");
}
}

@Test
@Tag("v2")
void shouldInsertAndSelectComplexStruct() throws SQLException {
Car car1 = Car.builder().ts(new Timestamp(2)).d(new Date(3)).tags(new String[] { "fast", "sleek" }).build();

executeStatementFromFile("/statements/prepared-statement-struct/ddl.sql");
try (Connection connection = createConnection()) {

try (PreparedStatement statement = connection
.prepareStatement("INSERT INTO test_struct_helper(a, b) VALUES (?,?)")) {
statement.setArray(1, connection.createArrayOf("VARCHAR", car1.getTags()));
statement.setTimestamp(2, car1.getTs());
statement.executeUpdate();
}

setParam(connection, "advanced_mode", "true");
setParam(connection, "enable_row_selection", "true");
try (Statement statement = connection.createStatement()) {
statement.execute(
"INSERT INTO test_struct(id, s) SELECT 1, test_struct_helper FROM test_struct_helper");
}
try (Statement statement = connection.createStatement();
ResultSet rs = statement
.executeQuery("SELECT test_struct FROM test_struct")) {
rs.next();
assertEquals(FireboltDataType.STRUCT.name().toLowerCase()
+ "(id int, s struct(a array(text null), b timestamp null))",
rs.getMetaData().getColumnTypeName(1).toLowerCase());
String expectedJson = String.format(
"{\"id\":%d,\"s\":{\"a\":[\"%s\",\"%s\"],\"b\":\"%s\"}}", 1, car1.getTags()[0],
car1.getTags()[1], car1.getTs().toString());
assertEquals(expectedJson, rs.getString(1));
}
} finally {
executeStatementFromFile("/statements/prepared-statement-struct/cleanup.sql");
}
}

private QueryResult createExpectedResult(List<List<?>> expectedRows) {
return QueryResult.builder().databaseName(ConnectionInfo.getInstance().getDatabase())
.tableName("prepared_statement_test")
Expand Down Expand Up @@ -493,6 +565,7 @@ private static class Car {
Timestamp ts;
Date d;
URL url;
String[] tags;
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
DROP TABLE IF EXISTS test_struct;
DROP TABLE IF EXISTS test_struct_helper;
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
SET advanced_mode=1;
SET enable_struct=1;
SET enable_create_table_v2=true;
SET enable_row_selection=true;
SET prevent_create_on_information_schema=true;
SET enable_create_table_with_struct_type=true;
DROP TABLE IF EXISTS test_struct;
DROP TABLE IF EXISTS test_struct_helper;
CREATE TABLE IF NOT EXISTS test_struct(id int not null, s struct(a array(text) not null, b datetime null) not null);
CREATE TABLE IF NOT EXISTS test_struct_helper(a array(text) not null, b datetime null);
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import java.util.stream.Collectors;

import static com.firebolt.jdbc.type.FireboltDataType.ARRAY;
import static com.firebolt.jdbc.type.FireboltDataType.STRUCT;
import static com.firebolt.jdbc.type.FireboltDataType.TUPLE;
import static com.firebolt.jdbc.type.FireboltDataType.ofType;

Expand All @@ -40,6 +41,8 @@ public class ColumnType {
private static final Set<String> TIMEZONES = Arrays.stream(TimeZone.getAvailableIDs())
.collect(Collectors.toCollection(HashSet::new));
private static final Pattern COMMA_WITH_SPACES = Pattern.compile("\\s*,\\s*");
// Regex to split on comma and ignoring commas that are between parenthesis
private static final String COMPLEX_TYPE_PATTERN = ",(?![^()]*\\))";
@EqualsAndHashCode.Exclude
String name;
FireboltDataType dataType;
Expand All @@ -62,6 +65,8 @@ public static ColumnType of(String columnType) {
innerDataTypes = getCollectionSubType(FireboltDataType.ARRAY, typeWithoutNullKeyword);
} else if (isType(FireboltDataType.TUPLE, typeWithoutNullKeyword)) {
innerDataTypes = getCollectionSubType(FireboltDataType.TUPLE, typeWithoutNullKeyword);
} else if (isType(FireboltDataType.STRUCT, typeWithoutNullKeyword)) {
innerDataTypes = getCollectionSubType(FireboltDataType.STRUCT, typeWithoutNullKeyword);
}

int typeEndIndex = getTypeEndPosition(typeWithoutNullKeyword);
Expand Down Expand Up @@ -106,8 +111,9 @@ private static List<ColumnType> getCollectionSubType(FireboltDataType fireboltDa
}

if (fireboltDataType.equals(TUPLE)) {
types = typeWithoutNullKeyword.split(",(?![^()]*\\))"); // Regex to split on comma and ignoring comma that are between
// parenthesis
types = typeWithoutNullKeyword.split(COMPLEX_TYPE_PATTERN);
} else if (fireboltDataType.equals(STRUCT)) {
types = typeWithoutNullKeyword.split(COMPLEX_TYPE_PATTERN);
} else {
types = new String[] {typeWithoutNullKeyword};
}
Expand Down Expand Up @@ -177,6 +183,8 @@ public String getCompactTypeName() {
return getArrayCompactTypeName();
} else if (isTuple()) {
return getTupleCompactTypeName(innerTypes);
} else if (isStruct()) {
return name;
} else {
return dataType.getDisplayName();
}
Expand Down Expand Up @@ -209,6 +217,10 @@ private boolean isTuple() {
return dataType.equals(TUPLE);
}

private boolean isStruct() {
return dataType.equals(STRUCT);
}

public ColumnType getArrayBaseColumnType() {
if (innerTypes == null || innerTypes.isEmpty()) {
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ public enum FireboltDataType {
TUPLE(Types.OTHER, FireboltDataTypeDisplayNames.TUPLE, BaseType.OBJECT, false, true, 0, 0, 0, false,"Tuple"),
BYTEA(Types.BINARY, FireboltDataTypeDisplayNames.BYTEA, BaseType.BYTEA, false, true, 0, 0, 0, false, "ByteA"),
GEOGRAPHY(Types.VARCHAR, FireboltDataTypeDisplayNames.GEOGRAPHY, BaseType.TEXT, false, false, 0, 0, 0, false,
"Geography");
"Geography"),
STRUCT(Types.VARCHAR, FireboltDataTypeDisplayNames.STRUCT, BaseType.TEXT, false, false, 0, 0, 0, false, "Struct");

private static final Map<String, FireboltDataType> typeNameOrAliasToType;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,5 @@ public class FireboltDataTypeDisplayNames {
static final String TUPLE = "tuple";
static final String BYTEA = "bytea";
static final String GEOGRAPHY = "geography";
static final String STRUCT = "struct";
}
Original file line number Diff line number Diff line change
@@ -1,18 +1,22 @@
package com.firebolt.jdbc.resultset;

import com.firebolt.jdbc.CheckedFunction;
import com.firebolt.jdbc.exception.FireboltException;
import com.firebolt.jdbc.statement.FireboltStatement;
import com.firebolt.jdbc.util.LoggerUtil;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.function.Executable;
import org.junitpioneer.jupiter.DefaultTimeZone;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.junit.jupiter.MockitoExtension;
import static com.firebolt.jdbc.exception.ExceptionType.TYPE_TRANSFORMATION_ERROR;
import static java.lang.String.format;
import static java.sql.ResultSet.TYPE_FORWARD_ONLY;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;

import java.io.ByteArrayInputStream;
import java.io.IOException;
Expand Down Expand Up @@ -48,23 +52,20 @@
import java.util.TimeZone;
import java.util.concurrent.Callable;

import static com.firebolt.jdbc.exception.ExceptionType.TYPE_TRANSFORMATION_ERROR;
import static java.lang.String.format;
import static java.sql.ResultSet.TYPE_FORWARD_ONLY;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
import org.apache.commons.io.IOUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.junit.jupiter.api.function.Executable;
import org.junitpioneer.jupiter.DefaultTimeZone;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.junit.jupiter.MockitoExtension;

import com.firebolt.jdbc.CheckedFunction;
import com.firebolt.jdbc.exception.FireboltException;
import com.firebolt.jdbc.statement.FireboltStatement;
import com.firebolt.jdbc.util.LoggerUtil;

@ExtendWith(MockitoExtension.class)
@DefaultTimeZone("UTC")
Expand Down Expand Up @@ -1478,6 +1479,31 @@ void shouldReturnGeography() throws SQLException {
assertEquals(Types.VARCHAR, resultSet.getMetaData().getColumnType(9));
}

@Test
void shouldReturnStruct() throws SQLException {
inputStream = getInputStreamWithStruct();
resultSet = createResultSet(inputStream);
resultSet.next();
assertEquals("{\"a\":null}", resultSet.getObject(2));
assertEquals("{\"a\":null}", resultSet.getObject("null_struct"));
assertEquals("{\"a\":\"1\"}", resultSet.getObject(4));
assertEquals("{\"a\":\"1\"}", resultSet.getObject("a_struct"));
assertEquals("{\"a\":[1,2,3]}", resultSet.getObject(5));
assertEquals("{\"a\":[1,2,3]}", resultSet.getObject("array_struct"));
assertEquals("{\"x\":\"2\",\"a\":{\"b\":\"1\",\"c\":\"3\"}}", resultSet.getObject(6));
assertEquals("{\"x\":\"2\",\"a\":{\"b\":\"1\",\"c\":\"3\"}}", resultSet.getObject("nested_struct"));
// Returns native JDBC type
for (int i = 2; i <= 6; i++) {
assertEquals(Types.VARCHAR, resultSet.getMetaData().getColumnType(i));
}

assertEquals("STRUCT(A INT NULL)", resultSet.getMetaData().getColumnTypeName(2));
assertEquals("STRUCT(A INT)", resultSet.getMetaData().getColumnTypeName(3));
assertEquals("STRUCT(A INT)", resultSet.getMetaData().getColumnTypeName(4));
assertEquals("STRUCT(A ARRAY(INT))", resultSet.getMetaData().getColumnTypeName(5));
assertEquals("STRUCT(X INT, A STRUCT(B INT, C INT))", resultSet.getMetaData().getColumnTypeName(6));
}

@Test
void shouldBeCaseInsensitive() throws SQLException {
inputStream = getInputStreamWithCommonResponseExample();
Expand Down Expand Up @@ -1552,6 +1578,10 @@ private InputStream getInputStreamWithArray() {
return FireboltResultSetTest.class.getResourceAsStream("/responses/firebolt-response-with-array");
}

private InputStream getInputStreamWithStruct() {
return FireboltResultSetTest.class.getResourceAsStream("/responses/firebolt-response-with-struct-nofalse");
}

private ResultSet createResultSet(InputStream is) throws SQLException {
return new FireboltResultSet(is, "a_table", "a_db", 65535, false, fireboltStatement, true);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
id null_struct an_empty_struct a_struct array_struct nested_struct
Int64 struct(a int null) struct(a int) struct(a int) struct(a array(int)) struct(x int, a struct(b int, c int))
1 {"a":null} {"a":"1"} {"a":[1,2,3]} {"x":"2","a":{"b":"1","c":"3"}}
2 {"a":null} {"a":"2"} {"a":[1,2,3]} {"x":"2","a":{"b":"1","c":"3"}}
Loading