bump sdk version, prepare for columnar tables support

ydb-platform · Nov 16, 2024 · e364627 · e364627
1 parent b643e28
commit e364627
Show file tree

Hide file tree

Showing 5 changed files with 31 additions and 9 deletions.
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,24 @@
     <version>1.3-SNAPSHOT</version>
     <!-- <version>X.Y[-SNAPSHOT]</version> -->
     <packaging>jar</packaging>
+    <licenses>
+        <license>
+            <name>Apache License, Version 2.0</name>
+            <url>https://www.apache.org/licenses/LICENSE-2.0</url>
+        </license>
+    </licenses>
+    <scm>
+        <url>https://github.com/ydb-platform/ydb-spark-connector</url>
+        <connection>scm:git:https://github.com/ydb-platform/ydb-spark-connector.git</connection>
+        <developerConnection>scm:git:https://github.com/ydb-platform/ydb-spark-connector.git</developerConnection>
+    </scm>
     <developers>
+        <developer>
+            <name>Aleksandr Gorshenin</name>
+            <email>[email protected]</email>
+            <organization>YDB</organization>
+            <organizationUrl>https://ydb.tech/</organizationUrl>
+        </developer>
         <developer>
             <name>Maksim Zinal</name>
             <email>[email protected]</email>
@@ -27,11 +44,11 @@
             <dependency>
                 <groupId>tech.ydb</groupId>
                 <artifactId>ydb-sdk-bom</artifactId>
-                <version>2.2.11-SNAPSHOT</version>
+                <version>2.3.6</version>
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
-            <dependency>
+            <dependency> <!-- for manual dependency injection in shaded mode -->
                 <groupId>io.grpc</groupId>
                 <artifactId>grpc-bom</artifactId>
                 <version>1.59.1</version>

diff --git a/src/main/java/tech/ydb/spark/connector/YdbPartitionReaderFactory.java b/src/main/java/tech/ydb/spark/connector/YdbPartitionReaderFactory.java
@@ -7,7 +7,7 @@
 import org.apache.spark.sql.connector.read.PartitionReader;
 import org.apache.spark.sql.connector.read.PartitionReaderFactory;
 
-import tech.ydb.spark.connector.impl.YdbScanImpl;
+import tech.ydb.spark.connector.impl.YdbScanReadTable;
 
 /**
  * Partition reader factory delivers the scan options to partition reader instances.
@@ -36,7 +36,7 @@ static class YdbReader implements PartitionReader<InternalRow> {
 
         private final YdbScanOptions options;
         private final YdbTablePartition partition;
-        private YdbScanImpl scan;
+        private YdbScanReadTable scan;
 
         YdbReader(YdbScanOptions options, YdbTablePartition partition) {
             this.options = options;
@@ -48,7 +48,7 @@ public boolean next() throws IOException {
             if (scan == null) {
                 LOG.debug("Preparing scan for table {} at partition {}",
                         options.getTablePath(), partition);
-                scan = new YdbScanImpl(options, partition.getRange());
+                scan = new YdbScanReadTable(options, partition.getRange());
                 scan.prepare();
                 LOG.debug("Scan prepared, ready to fetch...");
             }

diff --git a/src/main/java/tech/ydb/spark/connector/YdbTable.java b/src/main/java/tech/ydb/spark/connector/YdbTable.java
@@ -207,6 +207,7 @@ public static Result<YdbTable> lookup(YdbConnector connector, YdbTypes types,
     }
 
     static YdbStoreType convertStoreType(TableDescription td) {
+        /* TODO: implement store type detection
         switch (td.getStoreType()) {
             case COLUMN:
                 return YdbStoreType.COLUMN;
@@ -215,6 +216,8 @@ static YdbStoreType convertStoreType(TableDescription td) {
             default:
                 return YdbStoreType.UNSPECIFIED;
         }
+        */
+        return YdbStoreType.ROW;
     }
 
     static void fillProperties(Map<String, String> props, String tablePath,

diff --git a/src/main/java/tech/ydb/spark/connector/impl/YdbCreateTable.java b/src/main/java/tech/ydb/spark/connector/impl/YdbCreateTable.java
@@ -82,13 +82,14 @@ public CompletableFuture<Status> createTable(Session session) {
         }
         ps.setMinPartitionsCount(minPartitions);
         ps.setMaxPartitionsCount(maxPartitions);
-        long minSizeMb = getLongOption(YdbOptions.AP_PART_SIZE_MB, 2000L);
+        long minSizeMb = getLongOption(YdbOptions.AP_PART_SIZE_MB, 1000L);
         if (minSizeMb < 1L) {
             minSizeMb = 10L;
         }
         ps.setPartitionSize(minSizeMb);
         tdb.setPartitioningSettings(ps);
 
+        /* TODO: implement store type configuration
         switch (storeType) {
             case ROW:
                 tdb.setStoreType(TableDescription.StoreType.ROW);
@@ -99,6 +100,7 @@ public CompletableFuture<Status> createTable(Session session) {
             default:
                 break;
         }
+        */
 
         return session.createTable(tablePath, tdb.build());
     }

diff --git a/...ydb/spark/connector/impl/YdbScanImpl.java → ...park/connector/impl/YdbScanReadTable.java b/...ydb/spark/connector/impl/YdbScanImpl.java → ...park/connector/impl/YdbScanReadTable.java
@@ -28,10 +28,10 @@
  *
  * @author zinal
  */
-public class YdbScanImpl implements AutoCloseable {
+public class YdbScanReadTable implements AutoCloseable {
 
     private static final org.slf4j.Logger LOG
-            = org.slf4j.LoggerFactory.getLogger(YdbScanImpl.class);
+            = org.slf4j.LoggerFactory.getLogger(YdbScanReadTable.class);
 
     private static final QueueItem END_OF_SCAN = new QueueItem(null);
 
@@ -48,7 +48,7 @@ public class YdbScanImpl implements AutoCloseable {
     private volatile GrpcReadStream<ReadTablePart> stream;
     private ResultSetReader current;
 
-    public YdbScanImpl(YdbScanOptions options, YdbKeyRange keyRange) {
+    public YdbScanReadTable(YdbScanOptions options, YdbKeyRange keyRange) {
         this.options = options;
         this.keyRange = keyRange;
         this.queue = new ArrayBlockingQueue<>(options.getScanQueueDepth());