diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml new file mode 100644 index 000000000000..5c344e08b70f --- /dev/null +++ b/.github/workflows/dependency-check.yml @@ -0,0 +1,59 @@ +# This workflow will check if dependencies have changed (adding new dependencies or removing existing ones) + +name: Dependency Check + +on: + push: + branches: + - master + - 'rel/*' + - "rc/*" + paths-ignore: + - 'docs/**' + - 'site/**' + pull_request: + branches: + - master + - 'rel/*' + - "rc/*" + paths-ignore: + - 'docs/**' + - 'site/**' + # allow manually run the action: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 + MAVEN_ARGS: --batch-mode --no-transfer-progress + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + +jobs: + dependency-check: + strategy: + fail-fast: false + max-parallel: 15 + matrix: + java: [ 17 ] + os: [ ubuntu-latest ] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + - name: Set up JDK ${{ matrix.java }} + uses: actions/setup-java@v4 + with: + distribution: corretto + java-version: ${{ matrix.java }} + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2- + - name: Do the dependency check + shell: bash + run: mvn verify -Dmaven.test.skip=true -Dmdep.analyze.skip=true -P enable-sbom-check diff --git a/dependencies.json b/dependencies.json new file mode 100644 index 000000000000..0c8ff00acdd6 --- /dev/null +++ b/dependencies.json @@ -0,0 +1,167 @@ +{ + "dependencies": [ + "cglib:cglib", + "ch.qos.logback:logback-classic", + "ch.qos.logback:logback-core", + "ch.qos.reload4j:reload4j", + "com.bugsnag:bugsnag", + "com.digitalpetri.fsm:strict-machine", + "com.digitalpetri.netty:netty-channel-fsm", + "com.fasterxml.jackson.core:jackson-annotations", + "com.fasterxml.jackson.core:jackson-core", + "com.fasterxml.jackson.core:jackson-databind", + "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml", + "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", + "com.fasterxml.jackson.jaxrs:jackson-jaxrs-base", + "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider", + "com.fasterxml.jackson.module:jackson-module-jaxb-annotations", + "com.github.ben-manes.caffeine:caffeine", + "com.github.luben:zstd-jni", + "com.github.stephenc.jcip:jcip-annotations", + "com.github.wendykierp:JTransforms", + "com.google.code.findbugs:jsr305", + "com.google.code.gson:gson", + "com.google.errorprone:error_prone_annotations", + "com.google.guava:failureaccess", + "com.google.guava:guava", + "com.google.guava:listenablefuture", + "com.google.j2objc:j2objc-annotations", + "com.h2database:h2-mvstore", + "com.librato.metrics:librato-java", + "com.librato.metrics:metrics-librato", + "com.lmax:disruptor", + "com.nimbusds:content-type", + "com.nimbusds:lang-tag", + "com.nimbusds:nimbus-jose-jwt", + "com.nimbusds:oauth2-oidc-sdk", + "com.sun.istack:istack-commons-runtime", + "com.zaxxer:HikariCP", + "commons-cli:commons-cli", + "commons-codec:commons-codec", + "commons-io:commons-io", + "commons-logging:commons-logging", + "io.airlift:airline", + "io.airlift:concurrent", + "io.airlift:log", + "io.airlift:units", + "io.dropwizard.metrics:metrics-core", + "io.dropwizard.metrics:metrics-jvm", + "io.jsonwebtoken:jjwt-api", + "io.micrometer:micrometer-commons", + "io.micrometer:micrometer-core", + "io.micrometer:micrometer-observation", + "io.moquette:moquette-broker", + "io.netty:netty-buffer", + "io.netty:netty-codec", + "io.netty:netty-codec-dns", + "io.netty:netty-codec-http", + "io.netty:netty-codec-http2", + "io.netty:netty-codec-mqtt", + "io.netty:netty-codec-socks", + "io.netty:netty-common", + "io.netty:netty-handler", + "io.netty:netty-handler-proxy", + "io.netty:netty-resolver", + "io.netty:netty-resolver-dns", + "io.netty:netty-resolver-dns-classes-macos", + "io.netty:netty-resolver-dns-native-macos", + "io.netty:netty-transport", + "io.netty:netty-transport-classes-epoll", + "io.netty:netty-transport-native-epoll", + "io.netty:netty-transport-native-unix-common", + "io.projectreactor:reactor-core", + "io.projectreactor.netty:reactor-netty-core", + "io.projectreactor.netty:reactor-netty-http", + "io.swagger:swagger-annotations", + "io.swagger:swagger-core", + "io.swagger:swagger-jaxrs", + "io.swagger:swagger-models", + "jakarta.activation:jakarta.activation-api", + "jakarta.annotation:jakarta.annotation-api", + "jakarta.servlet:jakarta.servlet-api", + "jakarta.validation:jakarta.validation-api", + "jakarta.ws.rs:jakarta.ws.rs-api", + "jakarta.xml.bind:jakarta.xml.bind-api", + "net.java.dev.jna:jna", + "net.minidev:accessors-smart", + "net.minidev:json-smart", + "org.antlr:antlr4-runtime", + "org.apache.commons:commons-collections4", + "org.apache.commons:commons-csv", + "org.apache.commons:commons-jexl3", + "org.apache.commons:commons-lang3", + "org.apache.commons:commons-math3", + "org.apache.commons:commons-pool2", + "org.apache.httpcomponents:httpclient", + "org.apache.httpcomponents:httpcore", + "org.apache.ratis:ratis-client", + "org.apache.ratis:ratis-common", + "org.apache.ratis:ratis-grpc", + "org.apache.ratis:ratis-metrics-api", + "org.apache.ratis:ratis-proto", + "org.apache.ratis:ratis-server", + "org.apache.ratis:ratis-server-api", + "org.apache.ratis:ratis-thirdparty-misc", + "org.apache.thrift:libthrift", + "org.apache.tsfile:common", + "org.apache.tsfile:tsfile", + "org.bouncycastle:bcpkix-jdk18on", + "org.bouncycastle:bcprov-jdk18on", + "org.bouncycastle:bcutil-jdk18on", + "org.checkerframework:checker-qual", + "org.eclipse.collections:eclipse-collections", + "org.eclipse.collections:eclipse-collections-api", + "org.eclipse.jetty:jetty-http", + "org.eclipse.jetty:jetty-io", + "org.eclipse.jetty:jetty-security", + "org.eclipse.jetty:jetty-server", + "org.eclipse.jetty:jetty-servlet", + "org.eclipse.jetty:jetty-util", + "org.eclipse.jetty:jetty-util-ajax", + "org.eclipse.milo:bsd-core", + "org.eclipse.milo:bsd-generator", + "org.eclipse.milo:sdk-client", + "org.eclipse.milo:sdk-core", + "org.eclipse.milo:sdk-server", + "org.eclipse.milo:stack-client", + "org.eclipse.milo:stack-core", + "org.eclipse.milo:stack-server", + "org.fusesource.hawtbuf:hawtbuf", + "org.fusesource.hawtdispatch:hawtdispatch", + "org.fusesource.hawtdispatch:hawtdispatch-transport", + "org.fusesource.mqtt-client:mqtt-client", + "org.glassfish.hk2:hk2-api", + "org.glassfish.hk2:hk2-locator", + "org.glassfish.hk2:hk2-utils", + "org.glassfish.hk2:osgi-resource-locator", + "org.glassfish.hk2.external:aopalliance-repackaged", + "org.glassfish.hk2.external:jakarta.inject", + "org.glassfish.jaxb:jaxb-runtime", + "org.glassfish.jaxb:txw2", + "org.glassfish.jersey.containers:jersey-container-servlet-core", + "org.glassfish.jersey.core:jersey-client", + "org.glassfish.jersey.core:jersey-common", + "org.glassfish.jersey.core:jersey-server", + "org.glassfish.jersey.inject:jersey-hk2", + "org.glassfish.jersey.media:jersey-media-multipart", + "org.hdrhistogram:HdrHistogram", + "org.java-websocket:Java-WebSocket", + "org.javassist:javassist", + "org.jline:jline", + "org.jvnet.mimepull:mimepull", + "org.latencyutils:LatencyUtils", + "org.lz4:lz4-java", + "org.ops4j.pax.jdbc:pax-jdbc-common", + "org.osgi:osgi.cmpn", + "org.osgi:osgi.core", + "org.ow2.asm:asm", + "org.reactivestreams:reactive-streams", + "org.reflections:reflections", + "org.slf4j:slf4j-api", + "org.slf4j:slf4j-reload4j", + "org.tukaani:xz", + "org.xerial.snappy:snappy-java", + "org.yaml:snakeyaml", + "pl.edu.icm:JLargeArrays" + ] +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java index bb3a8de7d12f..620043e6c7b7 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java @@ -128,5 +128,14 @@ public void testBigDateTime() { e.printStackTrace(); fail(); } + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("insert into root.sg.d1(time,s2) values (16182830055860000000, 8.76);"); + fail(); + } catch (SQLException e) { + Assert.assertTrue( + e.getMessage() + .contains("please check whether the timestamp 16182830055860000000 is correct.")); + } } } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/path/IoTDBQuotedPathIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/path/IoTDBQuotedPathIT.java index 0c788782f2ab..830c0a06d279 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/path/IoTDBQuotedPathIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/path/IoTDBQuotedPathIT.java @@ -101,15 +101,16 @@ public void test() { @Test public void testIllegalStorageGroup() { - try (Connection connection = EnvFactory.getEnv().getConnection(); - Statement statement = connection.createStatement()) { + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { statement.execute("CREATE DATABASE root.`\"ln`"); - } catch (SQLException e) { + } catch (final SQLException e) { Assert.assertTrue( e.getMessage().contains("Error StorageGroup name") || e.getMessage() - .contains("The database name can only be characters, numbers and underscores.")); - } catch (Exception e) { + .contains( + "The database name can only contain english or chinese characters, numbers, backticks and underscores.")); + } catch (final Exception e) { e.printStackTrace(); Assert.fail(); } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java index fb541e7dcdb7..364c5d475f0c 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java @@ -36,6 +36,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -470,7 +471,7 @@ public void testAlterPipeSourceAndProcessor() { try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { statement.execute(sql); - } catch (SQLException e) { + } catch (final SQLException e) { fail(e.getMessage()); } @@ -491,12 +492,13 @@ public void testAlterPipeSourceAndProcessor() { TestUtils.assertDataEventuallyOnEnv( receiverEnv, "select * from root.db.**", "Time,root.db.d1.at1,", expectedResSet); - // Alter pipe (modify 'source.path' and 'processor.tumbling-time.interval-seconds') + // Alter pipe (modify 'source.path', 'source.inclusion' and + // 'processor.tumbling-time.interval-seconds') try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { statement.execute( - "alter pipe a2b modify source('source' = 'iotdb-source','source.path'='root.db.d2.**') modify processor ('processor.tumbling-time.interval-seconds'='2')"); - } catch (SQLException e) { + "alter pipe a2b modify source('source' = 'iotdb-source','source.path'='root.db.d2.**', 'source.inclusion'='all') modify processor ('processor.tumbling-time.interval-seconds'='2')"); + } catch (final SQLException e) { fail(e.getMessage()); } @@ -527,5 +529,15 @@ public void testAlterPipeSourceAndProcessor() { "select * from root.db.** where time > 10000", "Time,root.db.d1.at1,root.db.d2.at1,", expectedResSet); + + // Create database on sender + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, "create timeSeries root.db.d2.at2 int32")) { + fail(); + } + + // Check database on receiver + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "count timeSeries", "count(timeseries),", Collections.singleton("3,")); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java index 348aebc6b5b3..feda4776749a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBMultiIDsWithAttributesTableIT.java @@ -389,6 +389,25 @@ public void projectSortTest() { DATABASE_NAME); } + @Test + public void coalesceTest() { + String[] expectedHeader = new String[] {"time", "level", "coalesce_attr2", "str"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.040Z,l3,a,apricot,", + "1970-01-01T00:00:00.040Z,l3,CCC,apricot,", + "1970-01-01T00:00:00.020Z,l2,CCC,pineapple,", + "1970-01-01T00:00:00.020Z,l2,zz,pineapple,", + "1970-01-01T00:00:00.000Z,l1,d,coconut,", + "1970-01-01T00:00:00.000Z,l1,c,coconut,", + }; + tableResultSetEqualTest( + "select time,level,coalesce(attr2, 'CCC', 'DDD') as coalesce_attr2,str from table0 order by num+1,attr1 limit 6", + expectedHeader, + retArray, + DATABASE_NAME); + } + // ========== SubQuery Test ========= @Test public void subQueryTest1() { @@ -528,5 +547,171 @@ public void innerJoinTest2() { + "AND t2.time<=31536001000 AND t2.device in ('d1','d2')\n" + "ORDER BY t1.time, t1.device, t2.device LIMIT 20"; tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // join using + sql = + "SELECT time, t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0 WHERE time>=80) t1 \n" + + "JOIN (SELECT * FROM table0 WHERE floatNum<1000) t2 USING(time) \n" + + "WHERE cast(t1.num as double)>0 AND t1.level!='l1' \n" + + "AND time<=31536001000 AND t2.device in ('d1','d2')\n" + + "ORDER BY time, t1.device, t2.device LIMIT 20"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + // no filter + @Test + public void fullOuterJoinTest1() { + String[] expectedHeader = + new String[] {"time", "device", "level", "num", "device", "attr2", "num", "str"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.000Z,d1,l1,3,d1,d,3,coconut,", + "1970-01-01T00:00:00.000Z,d1,l1,3,d2,c,3,coconut,", + "1970-01-01T00:00:00.000Z,d2,l1,3,d1,d,3,coconut,", + "1970-01-01T00:00:00.000Z,d2,l1,3,d2,c,3,coconut,", + "1970-01-01T00:00:00.020Z,d1,l2,2,d1,zz,2,pineapple,", + "1970-01-01T00:00:00.020Z,d1,l2,2,d2,null,2,pineapple,", + "1970-01-01T00:00:00.020Z,d2,l2,2,d1,zz,2,pineapple,", + "1970-01-01T00:00:00.020Z,d2,l2,2,d2,null,2,pineapple,", + "1970-01-01T00:00:00.040Z,d1,l3,1,d1,a,1,apricot,", + "1970-01-01T00:00:00.040Z,d1,l3,1,d2,null,1,apricot,", + "1970-01-01T00:00:00.040Z,d2,l3,1,d1,a,1,apricot,", + "1970-01-01T00:00:00.040Z,d2,l3,1,d2,null,1,apricot,", + "1970-01-01T00:00:00.080Z,d1,l4,9,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d1,l4,9,d2,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,9,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,9,d2,null,9,apple,", + "1970-01-01T00:00:00.100Z,d1,l5,8,d1,null,8,papaya,", + "1970-01-01T00:00:00.100Z,d1,l5,8,d2,null,8,papaya,", + "1970-01-01T00:00:00.100Z,d2,l5,8,d1,null,8,papaya,", + "1970-01-01T00:00:00.100Z,d2,l5,8,d2,null,8,papaya,", + "1971-01-01T00:00:00.000Z,d1,l1,6,d1,d,6,banana,", + "1971-01-01T00:00:00.000Z,d1,l1,6,d2,c,6,banana,", + "1971-01-01T00:00:00.000Z,d2,l1,6,d1,d,6,banana,", + "1971-01-01T00:00:00.000Z,d2,l1,6,d2,c,6,banana,", + "1971-01-01T00:00:00.100Z,d1,l2,10,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d1,l2,10,d2,null,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,10,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,10,d2,null,10,pumelo,", + "1971-01-01T00:00:00.500Z,d1,l3,4,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d1,l3,4,d2,null,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,4,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,4,d2,null,4,peach,", + "1971-01-01T00:00:01.000Z,d1,l4,5,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d1,l4,5,d2,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,5,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,5,d2,null,5,orange,", + "1971-01-01T00:00:10.000Z,d1,l5,7,d1,null,7,lemon,", + "1971-01-01T00:00:10.000Z,d1,l5,7,d2,null,7,lemon,", + "1971-01-01T00:00:10.000Z,d2,l5,7,d1,null,7,lemon,", + "1971-01-01T00:00:10.000Z,d2,l5,7,d2,null,7,lemon,", + "1971-01-01T00:01:40.000Z,d1,l1,11,d1,d,11,pitaya,", + "1971-01-01T00:01:40.000Z,d1,l1,11,d2,c,11,pitaya,", + "1971-01-01T00:01:40.000Z,d2,l1,11,d1,d,11,pitaya,", + "1971-01-01T00:01:40.000Z,d2,l1,11,d2,c,11,pitaya,", + "1971-04-26T17:46:40.000Z,d1,l2,12,d1,zz,12,strawberry,", + "1971-04-26T17:46:40.000Z,d1,l2,12,d2,null,12,strawberry,", + "1971-04-26T17:46:40.000Z,d2,l2,12,d1,zz,12,strawberry,", + "1971-04-26T17:46:40.000Z,d2,l2,12,d2,null,12,strawberry,", + "1971-04-26T17:46:40.020Z,d1,l3,14,d1,a,14,cherry,", + "1971-04-26T17:46:40.020Z,d1,l3,14,d2,null,14,cherry,", + "1971-04-26T17:46:40.020Z,d2,l3,14,d1,a,14,cherry,", + "1971-04-26T17:46:40.020Z,d2,l3,14,d2,null,14,cherry,", + "1971-04-26T18:01:40.000Z,d1,l4,13,d1,null,13,lychee,", + "1971-04-26T18:01:40.000Z,d1,l4,13,d2,null,13,lychee,", + "1971-04-26T18:01:40.000Z,d2,l4,13,d1,null,13,lychee,", + "1971-04-26T18:01:40.000Z,d2,l4,13,d2,null,13,lychee,", + "1971-08-20T11:33:20.000Z,d1,l5,15,d1,null,15,watermelon,", + "1971-08-20T11:33:20.000Z,d1,l5,15,d2,null,15,watermelon,", + "1971-08-20T11:33:20.000Z,d2,l5,15,d1,null,15,watermelon,", + "1971-08-20T11:33:20.000Z,d2,l5,15,d2,null,15,watermelon,", + }; + + // join on + String sql = + "SELECT t1.time as time, t1.device, t1.level, t1.num, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM table0 t1 FULL JOIN table0 t2 ON t1.time = t2.time \n" + + "ORDER BY t1.time, t1.device, t2.device"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = + "SELECT t1.time as time, t1.device, t1.level, t1.num, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM table0 t1 INNER JOIN table0 t2 ON t1.time = t2.time \n" + + "ORDER BY t1.time, t1.device, t2.device"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // join using + sql = + "SELECT time, t1.device, t1.level, t1.num, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM table0 t1 FULL OUTER JOIN table0 t2 USING(time)\n" + + "ORDER BY time, t1.device, t2.device"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + sql = + "SELECT time, t1.device, t1.level, t1.num, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM table0 t1 INNER JOIN table0 t2 USING(time)\n" + + "ORDER BY time, t1.device, t2.device"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + } + + // has filter + @Test + public void fullOuterJoinTest2() { + String[] expectedHeader = + new String[] {"time", "device", "level", "t1_num_add", "device", "attr2", "num", "str"}; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.000Z,null,null,null,d1,d,3,coconut,", + "1970-01-01T00:00:00.000Z,null,null,null,d2,c,3,coconut,", + "1970-01-01T00:00:00.020Z,null,null,null,d1,zz,2,pineapple,", + "1970-01-01T00:00:00.020Z,null,null,null,d2,null,2,pineapple,", + "1970-01-01T00:00:00.040Z,null,null,null,d1,a,1,apricot,", + "1970-01-01T00:00:00.040Z,null,null,null,d2,null,1,apricot,", + "1970-01-01T00:00:00.080Z,d1,l4,10,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d1,l4,10,d2,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,10,d1,null,9,apple,", + "1970-01-01T00:00:00.080Z,d2,l4,10,d2,null,9,apple,", + "1971-01-01T00:00:00.100Z,d1,l2,11,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d1,l2,11,d2,null,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,11,d1,zz,10,pumelo,", + "1971-01-01T00:00:00.100Z,d2,l2,11,d2,null,10,pumelo,", + "1971-01-01T00:00:00.500Z,d1,l3,5,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d1,l3,5,d2,null,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,5,d1,a,4,peach,", + "1971-01-01T00:00:00.500Z,d2,l3,5,d2,null,4,peach,", + "1971-01-01T00:00:01.000Z,d1,l4,6,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d1,l4,6,d2,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,6,d1,null,5,orange,", + "1971-01-01T00:00:01.000Z,d2,l4,6,d2,null,5,orange,", + "1971-01-01T00:00:10.000Z,d1,l5,8,null,null,null,null,", + "1971-01-01T00:00:10.000Z,d2,l5,8,null,null,null,null,", + "1971-04-26T17:46:40.000Z,d1,l2,13,null,null,null,null,", + "1971-04-26T17:46:40.000Z,d2,l2,13,null,null,null,null,", + "1971-04-26T17:46:40.020Z,d1,l3,15,null,null,null,null,", + "1971-04-26T17:46:40.020Z,d2,l3,15,null,null,null,null,", + "1971-04-26T18:01:40.000Z,d1,l4,14,null,null,null,null,", + "1971-04-26T18:01:40.000Z,d2,l4,14,null,null,null,null,", + "1971-08-20T11:33:20.000Z,d1,l5,16,null,null,null,null,", + "1971-08-20T11:33:20.000Z,d2,l5,16,null,null,null,null,", + }; + + // join using + String sql = + "SELECT time, t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0 WHERE TIME>=80 AND level!='l1' AND cast(num as double)>0) t1 \n" + + "FULL JOIN (SELECT * FROM table0 WHERE TIME<=31536001000 AND floatNum<1000 AND device in ('d1','d2')) t2 \n" + + "USING(time) \n" + + "ORDER BY time, t1.device, t2.device"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); + + // join on + sql = + "SELECT COALESCE(t1.time, t2.time) as time, t1.device, t1.level, t1_num_add, t2.device, t2.attr2, t2.num, t2.str\n" + + "FROM (SELECT *,num+1 as t1_num_add FROM table0 WHERE TIME>=80 AND level!='l1' AND cast(num as double)>0) t1 \n" + + "FULL JOIN (SELECT * FROM table0 WHERE TIME<=31536001000 AND floatNum<1000 AND device in ('d1','d2')) t2 \n" + + "ON t1.time = t2.time \n" + + "ORDER BY time, t1.device, t2.device"; + tableResultSetEqualTest(sql, expectedHeader, retArray, DATABASE_NAME); } } diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/session/pool/IoTDBInsertTableSessionPoolIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/pool/IoTDBInsertTableSessionPoolIT.java new file mode 100644 index 000000000000..c4f95f846f2a --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/session/pool/IoTDBInsertTableSessionPoolIT.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.session.pool; + +import org.apache.iotdb.isession.IPooledSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.isession.pool.ISessionPool; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.record.Tablet.ColumnType; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.nio.charset.Charset; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBInsertTableSessionPoolIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + ISessionPool sessionPool = EnvFactory.getEnv().getSessionPool(1, "table"); + try (final IPooledSession session = sessionPool.getPooledSession()) { + session.executeNonQueryStatement("create database if not exists test"); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testPartialInsertTablet() { + ISessionPool sessionPool = EnvFactory.getEnv().getSessionPool(1, "table"); + try (final IPooledSession session = sessionPool.getPooledSession()) { + session.executeNonQueryStatement("use \"test\""); + session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='false'"); + session.executeNonQueryStatement( + "create table sg6 (id1 string id, s1 int64 measurement, s2 int64 measurement)"); + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("id1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s3", TSDataType.INT64)); + final List columnTypes = + Arrays.asList( + ColumnType.ID, + ColumnType.MEASUREMENT, + ColumnType.MEASUREMENT, + ColumnType.MEASUREMENT); + Tablet tablet = new Tablet("sg6", schemaList, columnTypes, 300); + long timestamp = 0; + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + for (int s = 0; s < 4; s++) { + long value = timestamp; + if (s == 0) { + tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, "d1"); + } else { + tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + } + } + timestamp++; + } + timestamp = System.currentTimeMillis(); + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + for (int s = 0; s < 4; s++) { + long value = timestamp; + if (s == 0) { + tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, "d1"); + } else { + tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + } + } + timestamp++; + } + try { + session.insertTablet(tablet); + } catch (Exception e) { + if (!e.getMessage().contains("507")) { + fail(e.getMessage()); + } + } finally { + session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='false'"); + } + try (SessionDataSet dataSet = session.executeQueryStatement("SELECT * FROM sg6")) { + assertEquals(dataSet.getColumnNames().size(), 4); + assertEquals(dataSet.getColumnNames().get(0), "time"); + assertEquals(dataSet.getColumnNames().get(1), "id1"); + assertEquals(dataSet.getColumnNames().get(2), "s1"); + assertEquals(dataSet.getColumnNames().get(3), "s2"); + int cnt = 0; + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + long time = rowRecord.getFields().get(0).getLongV(); + assertEquals(time, rowRecord.getFields().get(2).getLongV()); + assertEquals(time, rowRecord.getFields().get(3).getLongV()); + cnt++; + } + Assert.assertEquals(200, cnt); + } + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + @Test + public void testInsertKeyword() throws IoTDBConnectionException, StatementExecutionException { + ISessionPool sessionPool = EnvFactory.getEnv().getSessionPool(1, "table"); + try (final IPooledSession session = sessionPool.getPooledSession()) { + session.executeNonQueryStatement("USE \"test\""); + session.executeNonQueryStatement( + "create table table20 (" + + "device_id string id," + + "attribute STRING ATTRIBUTE," + + "boolean boolean MEASUREMENT," + + "int32 int32 MEASUREMENT," + + "int64 int64 MEASUREMENT," + + "float float MEASUREMENT," + + "double double MEASUREMENT," + + "text text MEASUREMENT," + + "string string MEASUREMENT," + + "blob blob MEASUREMENT," + + "timestamp01 timestamp MEASUREMENT," + + "date date MEASUREMENT)"); + + List schemas = new ArrayList<>(); + schemas.add(new MeasurementSchema("device_id", TSDataType.STRING)); + schemas.add(new MeasurementSchema("attribute", TSDataType.STRING)); + schemas.add(new MeasurementSchema("boolean", TSDataType.BOOLEAN)); + schemas.add(new MeasurementSchema("int32", TSDataType.INT32)); + schemas.add(new MeasurementSchema("int64", TSDataType.INT64)); + schemas.add(new MeasurementSchema("float", TSDataType.FLOAT)); + schemas.add(new MeasurementSchema("double", TSDataType.DOUBLE)); + schemas.add(new MeasurementSchema("text", TSDataType.TEXT)); + schemas.add(new MeasurementSchema("string", TSDataType.STRING)); + schemas.add(new MeasurementSchema("blob", TSDataType.BLOB)); + schemas.add(new MeasurementSchema("timestamp", TSDataType.TIMESTAMP)); + schemas.add(new MeasurementSchema("date", TSDataType.DATE)); + final List columnTypes = + Arrays.asList( + ColumnType.ID, + ColumnType.ATTRIBUTE, + ColumnType.MEASUREMENT, + ColumnType.MEASUREMENT, + ColumnType.MEASUREMENT, + ColumnType.MEASUREMENT, + ColumnType.MEASUREMENT, + ColumnType.MEASUREMENT, + ColumnType.MEASUREMENT, + ColumnType.MEASUREMENT, + ColumnType.MEASUREMENT, + ColumnType.MEASUREMENT); + + long timestamp = 0; + Tablet tablet = new Tablet("table20", schemas, columnTypes, 10); + + for (long row = 0; row < 10; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("device_id", rowIndex, "1"); + tablet.addValue("attribute", rowIndex, "1"); + tablet.addValue("boolean", rowIndex, true); + tablet.addValue("int32", rowIndex, Integer.valueOf("1")); + tablet.addValue("int64", rowIndex, Long.valueOf("1")); + tablet.addValue("float", rowIndex, Float.valueOf("1.0")); + tablet.addValue("double", rowIndex, Double.valueOf("1.0")); + tablet.addValue("text", rowIndex, "true"); + tablet.addValue("string", rowIndex, "true"); + tablet.addValue("blob", rowIndex, new Binary("iotdb", Charset.defaultCharset())); + tablet.addValue("timestamp", rowIndex, 1L); + tablet.addValue("date", rowIndex, LocalDate.parse("2024-08-15")); + } + session.insertTablet(tablet); + + SessionDataSet rs1 = + session.executeQueryStatement( + "select time, device_id, attribute, boolean, int32, int64, float, double, text, string, blob, timestamp, date from table20 order by time"); + for (int i = 0; i < 10; i++) { + RowRecord rec = rs1.next(); + assertEquals(i, rec.getFields().get(0).getLongV()); + assertEquals("1", rec.getFields().get(1).getStringValue()); + assertEquals("1", rec.getFields().get(2).getStringValue()); + assertTrue(rec.getFields().get(3).getBoolV()); + assertEquals(1, rec.getFields().get(4).getIntV()); + assertEquals(1, rec.getFields().get(5).getLongV()); + assertEquals(1.0, rec.getFields().get(6).getFloatV(), 0.001); + assertEquals(1.0, rec.getFields().get(7).getDoubleV(), 0.001); + assertEquals("true", rec.getFields().get(8).getStringValue()); + assertEquals("true", rec.getFields().get(9).getStringValue()); + assertEquals("0x696f746462", rec.getFields().get(10).getStringValue()); + assertEquals(1, rec.getFields().get(11).getLongV()); + assertEquals("20240815", rec.getFields().get(12).getStringValue()); + } + assertFalse(rs1.hasNext()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java index fc7c96daeffa..d77b2c819072 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionRelationalIT.java @@ -674,6 +674,82 @@ public void insertRelationalTabletTest() } } + @Test + @Category({LocalStandaloneIT.class, ClusterIT.class}) + public void insertRelationalTabletWithCacheLeaderTest() + throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection(TABLE_SQL_DIALECT)) { + session.executeNonQueryStatement("USE \"db1\""); + session.executeNonQueryStatement( + "CREATE TABLE table5 (id1 string id, attr1 string attribute, " + + "m1 double " + + "measurement)"); + + List schemaList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("id1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("attr1", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("m1", TSDataType.DOUBLE)); + final List columnTypes = + Arrays.asList(ColumnType.ID, ColumnType.ATTRIBUTE, ColumnType.MEASUREMENT); + + long timestamp = 0; + Tablet tablet = new Tablet("table5", schemaList, columnTypes, 15); + + for (long row = 0; row < 15; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp + row); + tablet.addValue("id1", rowIndex, "id:" + row); + tablet.addValue("attr1", rowIndex, "attr:" + row); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.rowSize == tablet.getMaxRowNumber()) { + session.insertRelationalTablet(tablet, true); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session.insertRelationalTablet(tablet); + tablet.reset(); + } + + session.executeNonQueryStatement("FLush"); + + for (long row = 15; row < 30; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp + row); + // cache leader should work for devices that have inserted before + tablet.addValue("id1", rowIndex, "id:" + (row - 15)); + tablet.addValue("attr1", rowIndex, "attr:" + (row - 15)); + tablet.addValue("m1", rowIndex, row * 1.0); + if (tablet.rowSize == tablet.getMaxRowNumber()) { + session.insertRelationalTablet(tablet, true); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session.insertRelationalTablet(tablet); + tablet.reset(); + } + + int cnt = 0; + SessionDataSet dataSet = session.executeQueryStatement("select * from table5 order by time"); + while (dataSet.hasNext()) { + RowRecord rowRecord = dataSet.next(); + timestamp = rowRecord.getFields().get(0).getLongV(); + assertEquals( + "id:" + (timestamp < 15 ? timestamp : timestamp - 15), + rowRecord.getFields().get(1).getBinaryV().toString()); + assertEquals( + "attr:" + (timestamp < 15 ? timestamp : timestamp - 15), + rowRecord.getFields().get(2).getBinaryV().toString()); + assertEquals(timestamp * 1.0, rowRecord.getFields().get(3).getDoubleV(), 0.0001); + cnt++; + } + assertEquals(30, cnt); + } + } + @Test @Category({LocalStandaloneIT.class, ClusterIT.class}) public void autoCreateTableTest() throws IoTDBConnectionException, StatementExecutionException { diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ExportTsFile.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ExportTsFile.java index 22cdcb907da3..0a6586863b1d 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ExportTsFile.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ExportTsFile.java @@ -55,10 +55,12 @@ import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; public class ExportTsFile extends AbstractTsFileTool { @@ -299,7 +301,7 @@ private static void dumpResult(String sql, int index) { final String path = targetDirectory + targetFile + index + ".tsfile"; try (SessionDataSet sessionDataSet = session.executeQueryStatement(sql, timeout)) { long start = System.currentTimeMillis(); - writeTsFileFile(sessionDataSet, path); + writeWithTablets(sessionDataSet, path); long end = System.currentTimeMillis(); ioTPrinter.println("Export completely!cost: " + (end - start) + " ms."); } catch (StatementExecutionException @@ -310,11 +312,119 @@ private static void dumpResult(String sql, int index) { } } + private static void collectSchemas( + List columnNames, + List columnTypes, + Map> deviceSchemaMap, + Set alignedDevices, + Map> deviceColumnIndices) + throws IoTDBConnectionException, StatementExecutionException { + for (int i = 0; i < columnNames.size(); i++) { + String column = columnNames.get(i); + if (!column.startsWith("root.")) { + continue; + } + TSDataType tsDataType = getTsDataType(columnTypes.get(i)); + Path path = new Path(column, true); + String deviceId = path.getDeviceString(); + // query whether the device is aligned or not + try (SessionDataSet deviceDataSet = + session.executeQueryStatement("show devices " + deviceId, timeout)) { + List deviceList = deviceDataSet.next().getFields(); + if (deviceList.size() > 1 && "true".equals(deviceList.get(1).getStringValue())) { + alignedDevices.add(deviceId); + } + } + + // query timeseries metadata + MeasurementSchema measurementSchema = + new MeasurementSchema(path.getMeasurement(), tsDataType); + List seriesList = + session.executeQueryStatement("show timeseries " + column, timeout).next().getFields(); + measurementSchema.setEncoding( + TSEncoding.valueOf(seriesList.get(4).getStringValue()).serialize()); + measurementSchema.setCompressor( + CompressionType.valueOf(seriesList.get(5).getStringValue()).serialize()); + + deviceSchemaMap.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(measurementSchema); + deviceColumnIndices.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(i); + } + } + + private static List constructTablets( + Map> deviceSchemaMap, + Set alignedDevices, + TsFileWriter tsFileWriter) + throws WriteProcessException { + List tabletList = new ArrayList<>(deviceSchemaMap.size()); + for (Map.Entry> stringListEntry : deviceSchemaMap.entrySet()) { + String deviceId = stringListEntry.getKey(); + List schemaList = stringListEntry.getValue(); + Tablet tablet = new Tablet(deviceId, schemaList); + tablet.initBitMaps(); + Path path = new Path(tablet.getDeviceId()); + if (alignedDevices.contains(tablet.getDeviceId())) { + tsFileWriter.registerAlignedTimeseries(path, schemaList); + } else { + tsFileWriter.registerTimeseries(path, schemaList); + } + tabletList.add(tablet); + } + return tabletList; + } + + private static void writeWithTablets( + SessionDataSet sessionDataSet, + List tabletList, + Set alignedDevices, + TsFileWriter tsFileWriter, + Map> deviceColumnIndices) + throws IoTDBConnectionException, + StatementExecutionException, + IOException, + WriteProcessException { + while (sessionDataSet.hasNext()) { + RowRecord rowRecord = sessionDataSet.next(); + List fields = rowRecord.getFields(); + + for (Tablet tablet : tabletList) { + String deviceId = tablet.getDeviceId(); + List columnIndices = deviceColumnIndices.get(deviceId); + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, rowRecord.getTimestamp()); + List schemas = tablet.getSchemas(); + + for (int i = 0, columnIndicesSize = columnIndices.size(); i < columnIndicesSize; i++) { + Integer columnIndex = columnIndices.get(i); + IMeasurementSchema measurementSchema = schemas.get(i); + // -1 for time not in fields + Object value = fields.get(columnIndex - 1).getObjectValue(measurementSchema.getType()); + if (value == null) { + tablet.bitMaps[i].mark(rowIndex); + } + tablet.addValue(measurementSchema.getMeasurementId(), rowIndex, value); + } + + if (tablet.rowSize == tablet.getMaxRowNumber()) { + writeToTsFile(alignedDevices, tsFileWriter, tablet); + tablet.initBitMaps(); + tablet.reset(); + } + } + } + + for (Tablet tablet : tabletList) { + if (tablet.rowSize != 0) { + writeToTsFile(alignedDevices, tsFileWriter, tablet); + } + } + } + @SuppressWarnings({ "squid:S3776", "squid:S6541" }) // Suppress high Cognitive Complexity warning, Suppress many task in one method warning - public static void writeTsFileFile(SessionDataSet sessionDataSet, String filePath) + public static void writeWithTablets(SessionDataSet sessionDataSet, String filePath) throws IOException, IoTDBConnectionException, StatementExecutionException, @@ -325,91 +435,32 @@ public static void writeTsFileFile(SessionDataSet sessionDataSet, String filePat if (f.exists()) { Files.delete(f.toPath()); } - HashSet deviceFilterSet = new HashSet<>(); + try (TsFileWriter tsFileWriter = new TsFileWriter(f)) { - Map> schemaMap = new LinkedHashMap<>(); - for (int i = 0; i < columnNames.size(); i++) { - String column = columnNames.get(i); - if (!column.startsWith("root.")) { - continue; - } - TSDataType tsDataType = getTsDataType(columnTypes.get(i)); - Path path = new Path(column, true); - String deviceId = path.getDeviceString(); - try (SessionDataSet deviceDataSet = - session.executeQueryStatement("show devices " + deviceId, timeout)) { - List deviceList = deviceDataSet.next().getFields(); - if (deviceList.size() > 1 && "true".equals(deviceList.get(1).getStringValue())) { - deviceFilterSet.add(deviceId); - } - } - MeasurementSchema measurementSchema = - new MeasurementSchema(path.getMeasurement(), tsDataType); + // device -> column indices in columnNames + Map> deviceColumnIndices = new HashMap<>(); + Set alignedDevices = new HashSet<>(); + Map> deviceSchemaMap = new LinkedHashMap<>(); - List seriesList = - session.executeQueryStatement("show timeseries " + column, timeout).next().getFields(); + collectSchemas( + columnNames, columnTypes, deviceSchemaMap, alignedDevices, deviceColumnIndices); + + List tabletList = constructTablets(deviceSchemaMap, alignedDevices, tsFileWriter); - measurementSchema.setEncoding( - TSEncoding.valueOf(seriesList.get(4).getStringValue()).serialize()); - measurementSchema.setCompressor( - CompressionType.valueOf(seriesList.get(5).getStringValue()).serialize()); - schemaMap.computeIfAbsent(deviceId, key -> new ArrayList<>()).add(measurementSchema); - } - List tabletList = new ArrayList<>(); - for (Map.Entry> stringListEntry : schemaMap.entrySet()) { - String deviceId = stringListEntry.getKey(); - List schemaList = stringListEntry.getValue(); - Tablet tablet = new Tablet(deviceId, schemaList); - tablet.initBitMaps(); - Path path = new Path(tablet.getDeviceId()); - if (deviceFilterSet.contains(tablet.getDeviceId())) { - tsFileWriter.registerAlignedTimeseries(path, schemaList); - } else { - tsFileWriter.registerTimeseries(path, schemaList); - } - tabletList.add(tablet); - } if (tabletList.isEmpty()) { ioTPrinter.println("!!!Warning:Tablet is empty,no data can be exported."); System.exit(CODE_ERROR); } - while (sessionDataSet.hasNext()) { - RowRecord rowRecord = sessionDataSet.next(); - List fields = rowRecord.getFields(); - int i = 0; - while (i < fields.size()) { - for (Tablet tablet : tabletList) { - int rowIndex = tablet.rowSize++; - tablet.addTimestamp(rowIndex, rowRecord.getTimestamp()); - List schemas = tablet.getSchemas(); - for (int j = 0; j < schemas.size(); j++) { - IMeasurementSchema measurementSchema = schemas.get(j); - Object value = fields.get(i).getObjectValue(measurementSchema.getType()); - if (value == null) { - tablet.bitMaps[j].mark(rowIndex); - } - tablet.addValue(measurementSchema.getMeasurementId(), rowIndex, value); - i++; - } - if (tablet.rowSize == tablet.getMaxRowNumber()) { - writeToTsfile(deviceFilterSet, tsFileWriter, tablet); - tablet.initBitMaps(); - tablet.reset(); - } - } - } - } - for (Tablet tablet : tabletList) { - if (tablet.rowSize != 0) { - writeToTsfile(deviceFilterSet, tsFileWriter, tablet); - } - } + + writeWithTablets( + sessionDataSet, tabletList, alignedDevices, tsFileWriter, deviceColumnIndices); + tsFileWriter.flushAllChunkGroups(); } } - private static void writeToTsfile( - HashSet deviceFilterSet, TsFileWriter tsFileWriter, Tablet tablet) + private static void writeToTsFile( + Set deviceFilterSet, TsFileWriter tsFileWriter, Tablet tablet) throws IOException, WriteProcessException { if (deviceFilterSet.contains(tablet.getDeviceId())) { tsFileWriter.writeAligned(tablet); diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileScanTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileScanTool.java index 4ec9f2290848..4891f4254861 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileScanTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/tsfile/ImportTsFileScanTool.java @@ -86,7 +86,9 @@ public static void setSourceFullPath(final String sourceFullPath) { } public static int getSourceFullPathLength() { - return ImportTsFileScanTool.sourceFullPath.length(); + return new File(sourceFullPath).isDirectory() + ? ImportTsFileScanTool.sourceFullPath.length() + : new File(ImportTsFileScanTool.sourceFullPath).getParent().length(); } public static int getTsFileQueueSize() { diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/IPooledSession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/IPooledSession.java index 04c25944a976..f8764f9a203d 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/IPooledSession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/IPooledSession.java @@ -28,6 +28,7 @@ import org.apache.thrift.TException; import org.apache.tsfile.write.record.Tablet; +/** NOTICE: IPooledSession is specific to the table model. */ public interface IPooledSession extends AutoCloseable { Version getVersion(); diff --git a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java index 7bc317318025..ed627da82ef6 100644 --- a/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java +++ b/iotdb-client/isession/src/main/java/org/apache/iotdb/isession/ISession.java @@ -31,6 +31,7 @@ import org.apache.thrift.TException; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.write.record.Tablet; @@ -65,6 +66,14 @@ void open( INodeSupplier nodeSupplier) throws IoTDBConnectionException; + void open( + boolean enableRPCCompression, + int connectionTimeoutInMs, + Map deviceIdToEndpoint, + Map tabletModelDeviceIdToEndpoint, + INodeSupplier nodeSupplier) + throws IoTDBConnectionException; + void close() throws IoTDBConnectionException; String getTimeZone(); diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RedirectException.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RedirectException.java index 8da65e8249c0..c7db354b5291 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RedirectException.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RedirectException.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import java.io.IOException; +import java.util.List; import java.util.Map; public class RedirectException extends IOException { @@ -29,17 +30,27 @@ public class RedirectException extends IOException { private final TEndPoint endPoint; private final Map deviceEndPointMap; + private final List endPointList; public RedirectException(TEndPoint endPoint) { super("later request in same group will be redirected to " + endPoint.toString()); this.endPoint = endPoint; this.deviceEndPointMap = null; + this.endPointList = null; } public RedirectException(Map deviceEndPointMap) { super("later request in same group will be redirected to " + deviceEndPointMap); this.endPoint = null; this.deviceEndPointMap = deviceEndPointMap; + this.endPointList = null; + } + + public RedirectException(List endPointList) { + super("later request in same group will be redirected to " + endPointList); + this.endPoint = null; + this.deviceEndPointMap = null; + this.endPointList = endPointList; } public TEndPoint getEndPoint() { @@ -49,4 +60,8 @@ public TEndPoint getEndPoint() { public Map getDeviceEndPointMap() { return deviceEndPointMap; } + + public List getEndPointList() { + return endPointList; + } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java index 1a7bc9970f6d..d41c1c8fd970 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/RpcUtils.java @@ -36,6 +36,7 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -112,6 +113,22 @@ public static void verifySuccessWithRedirection(TSStatus status) if (status.isSetRedirectNode()) { throw new RedirectException(status.getRedirectNode()); } + if (status.isSetSubStatus()) { // the resp of insertRelationalTablet may set subStatus + List statusSubStatus = status.getSubStatus(); + List endPointList = new ArrayList<>(statusSubStatus.size()); + int count = 0; + for (TSStatus subStatus : statusSubStatus) { + if (subStatus.isSetRedirectNode()) { + endPointList.add(subStatus.getRedirectNode()); + count++; + } else { + endPointList.add(null); + } + } + if (!endPointList.isEmpty() && count != 0) { + throw new RedirectException(endPointList); + } + } } public static void verifySuccessWithRedirectionForMultiDevices( diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java index 35229b541627..78ecd13242d4 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java @@ -63,6 +63,7 @@ import org.apache.thrift.TException; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.utils.Binary; @@ -164,6 +165,9 @@ public class Session implements ISession { @SuppressWarnings("squid:S3077") // Non-primitive fields should not be "volatile" protected volatile Map deviceIdToEndpoint; + @SuppressWarnings("squid:S3077") // Non-primitive fields should not be "volatile" + protected volatile Map tableModelDeviceIdToEndpoint; + @SuppressWarnings("squid:S3077") // Non-primitive fields should not be "volatile" protected volatile Map endPointToSessionConnection; @@ -535,6 +539,7 @@ public synchronized void open(boolean enableRPCCompression, int connectionTimeou isClosed = false; if (enableRedirection || enableQueryRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); endPointToSessionConnection = new ConcurrentHashMap<>(); endPointToSessionConnection.put(defaultEndPoint, defaultSessionConnection); } @@ -584,6 +589,33 @@ public synchronized void open( isClosed = false; if (enableRedirection || enableQueryRedirection) { this.deviceIdToEndpoint = deviceIdToEndpoint; + this.tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); + endPointToSessionConnection = new ConcurrentHashMap<>(); + endPointToSessionConnection.put(defaultEndPoint, defaultSessionConnection); + } + } + + @Override + public synchronized void open( + boolean enableRPCCompression, + int connectionTimeoutInMs, + Map deviceIdToEndpoint, + Map tableModelDeviceIdToEndpoint, + INodeSupplier nodesSupplier) + throws IoTDBConnectionException { + if (!isClosed) { + return; + } + + this.availableNodes = nodesSupplier; + this.enableRPCCompression = enableRPCCompression; + this.connectionTimeoutInMs = connectionTimeoutInMs; + defaultSessionConnection = constructSessionConnection(this, defaultEndPoint, zoneId); + defaultSessionConnection.setEnableRedirect(enableQueryRedirection); + isClosed = false; + if (enableRedirection || enableQueryRedirection) { + this.deviceIdToEndpoint = deviceIdToEndpoint; + this.tableModelDeviceIdToEndpoint = tableModelDeviceIdToEndpoint; endPointToSessionConnection = new ConcurrentHashMap<>(); endPointToSessionConnection.put(defaultEndPoint, defaultSessionConnection); } @@ -1321,6 +1353,18 @@ private SessionConnection getSessionConnection(String deviceId) { } } + private SessionConnection getSessionConnection(IDeviceID deviceId) { + TEndPoint endPoint; + if (enableRedirection + && tableModelDeviceIdToEndpoint != null + && (endPoint = tableModelDeviceIdToEndpoint.get(deviceId)) != null + && endPointToSessionConnection.containsKey(endPoint)) { + return endPointToSessionConnection.get(endPoint); + } else { + return defaultSessionConnection; + } + } + @Override public String getTimestampPrecision() throws TException { return defaultSessionConnection.getClient().getProperties().getTimestampPrecision(); @@ -1343,8 +1387,7 @@ public void removeBrokenSessionConnection(SessionConnection sessionConnection) { } } } - - if (deviceIdToEndpoint != null) { + if (deviceIdToEndpoint != null && !deviceIdToEndpoint.isEmpty()) { for (Iterator> it = deviceIdToEndpoint.entrySet().iterator(); it.hasNext(); ) { Entry entry = it.next(); @@ -1353,6 +1396,16 @@ public void removeBrokenSessionConnection(SessionConnection sessionConnection) { } } } + if (tableModelDeviceIdToEndpoint != null && !tableModelDeviceIdToEndpoint.isEmpty()) { + for (Iterator> it = + tableModelDeviceIdToEndpoint.entrySet().iterator(); + it.hasNext(); ) { + Entry entry = it.next(); + if (entry.getValue().equals(endPoint)) { + it.remove(); + } + } + } } } @@ -1362,7 +1415,6 @@ private void handleRedirection(String deviceId, TEndPoint endpoint) { if (endpoint.ip.equals("0.0.0.0")) { return; } - AtomicReference exceptionReference = new AtomicReference<>(); if (!deviceIdToEndpoint.containsKey(deviceId) || !deviceIdToEndpoint.get(deviceId).equals(endpoint)) { deviceIdToEndpoint.put(deviceId, endpoint); @@ -1374,7 +1426,6 @@ private void handleRedirection(String deviceId, TEndPoint endpoint) { try { return constructSessionConnection(this, endpoint, zoneId); } catch (IoTDBConnectionException ex) { - exceptionReference.set(ex); return null; } }); @@ -1384,6 +1435,32 @@ private void handleRedirection(String deviceId, TEndPoint endpoint) { } } + private void handleRedirection(IDeviceID deviceId, TEndPoint endpoint) { + if (enableRedirection) { + // no need to redirection + if (endpoint.ip.equals("0.0.0.0")) { + return; + } + if (!tableModelDeviceIdToEndpoint.containsKey(deviceId) + || !tableModelDeviceIdToEndpoint.get(deviceId).equals(endpoint)) { + tableModelDeviceIdToEndpoint.put(deviceId, endpoint); + } + SessionConnection connection = + endPointToSessionConnection.computeIfAbsent( + endpoint, + k -> { + try { + return constructSessionConnection(this, endpoint, zoneId); + } catch (IoTDBConnectionException ex) { + return null; + } + }); + if (connection == null) { + tableModelDeviceIdToEndpoint.remove(deviceId); + } + } + } + private void handleQueryRedirection(TEndPoint endPoint) throws IoTDBConnectionException { if (enableQueryRedirection) { AtomicReference exceptionReference = new AtomicReference<>(); @@ -2492,7 +2569,7 @@ private TSInsertStringRecordsOfOneDeviceReq genTSInsertStringRecordsOfOneDeviceR *

e.g. source: [1,2,3,4,5], index:[1,0,3,2,4], return : [2,1,4,3,5] * * @param source Input list - * @param index retuen order + * @param index return order * @param Input type * @return ordered list */ @@ -2679,11 +2756,20 @@ private void insertTabletInternal(Tablet tablet, TSInsertTabletReq request) @Override public void insertRelationalTablet(Tablet tablet, boolean sorted) throws IoTDBConnectionException, StatementExecutionException { - TSInsertTabletReq request = genTSInsertTabletReq(tablet, sorted, false); - request.setWriteToTable(true); - request.setColumnCategories( - tablet.getColumnTypes().stream().map(t -> (byte) t.ordinal()).collect(Collectors.toList())); - insertTabletInternal(tablet, request); + if (enableRedirection) { + insertRelationalTabletWithLeaderCache(tablet); + } else { + TSInsertTabletReq request = genTSInsertTabletReq(tablet, sorted, false); + request.setWriteToTable(true); + request.setColumnCategories( + tablet.getColumnTypes().stream() + .map(t -> (byte) t.ordinal()) + .collect(Collectors.toList())); + try { + defaultSessionConnection.insertTablet(request); + } catch (RedirectException ignored) { + } + } } /** @@ -2697,6 +2783,113 @@ public void insertRelationalTablet(Tablet tablet) insertRelationalTablet(tablet, false); } + private void insertRelationalTabletWithLeaderCache(Tablet tablet) + throws IoTDBConnectionException, StatementExecutionException { + Map relationalTabletGroup = new HashMap<>(); + if (tableModelDeviceIdToEndpoint.isEmpty()) { + relationalTabletGroup.put(defaultSessionConnection, tablet); + } else { + for (int i = 0; i < tablet.rowSize; i++) { + IDeviceID iDeviceID = tablet.getDeviceID(i); + final SessionConnection connection = getSessionConnection(iDeviceID); + int finalI = i; + relationalTabletGroup.compute( + connection, + (k, v) -> { + if (v == null) { + v = + new Tablet( + tablet.getTableName(), + tablet.getSchemas(), + tablet.getColumnTypes(), + tablet.rowSize); + } + for (int j = 0; j < v.getSchemas().size(); j++) { + v.addValue( + v.getSchemas().get(j).getMeasurementId(), + v.rowSize, + tablet.getValue(finalI, j)); + } + v.addTimestamp(v.rowSize, tablet.timestamps[finalI]); + v.rowSize++; + return v; + }); + } + } + insertRelationalTabletByGroup(relationalTabletGroup); + } + + @SuppressWarnings({ + "squid:S3776" + }) // ignore Cognitive Complexity of methods should not be too high + private void insertRelationalTabletByGroup(Map relationalTabletGroup) + throws IoTDBConnectionException, StatementExecutionException { + List> completableFutures = + relationalTabletGroup.entrySet().stream() + .map( + entry -> { + SessionConnection connection = entry.getKey(); + Tablet subTablet = entry.getValue(); + return CompletableFuture.runAsync( + () -> { + TSInsertTabletReq request = genTSInsertTabletReq(subTablet, false, false); + request.setWriteToTable(true); + request.setColumnCategories( + subTablet.getColumnTypes().stream() + .map(t -> (byte) t.ordinal()) + .collect(Collectors.toList())); + InsertConsumer insertConsumer = + SessionConnection::insertTablet; + try { + insertConsumer.insert(connection, request); + } catch (RedirectException e) { + List endPointList = e.getEndPointList(); + Map endPointMap = new HashMap<>(); + for (int i = 0; i < endPointList.size(); i++) { + if (endPointList.get(i) != null) { + endPointMap.put(subTablet.getDeviceID(i), endPointList.get(i)); + } + } + endPointMap.forEach(this::handleRedirection); + } catch (StatementExecutionException e) { + throw new CompletionException(e); + } catch (IoTDBConnectionException e) { + // remove the broken session + removeBrokenSessionConnection(connection); + try { + insertConsumer.insert(defaultSessionConnection, request); + } catch (IoTDBConnectionException | StatementExecutionException ex) { + throw new CompletionException(ex); + } catch (RedirectException ignored) { + } + } + }, + OPERATION_EXECUTOR); + }) + .collect(Collectors.toList()); + + StringBuilder errMsgBuilder = new StringBuilder(); + for (CompletableFuture completableFuture : completableFutures) { + try { + completableFuture.join(); + } catch (CompletionException completionException) { + Throwable cause = completionException.getCause(); + logger.error("Meet error when async insert!", cause); + if (cause instanceof IoTDBConnectionException) { + throw (IoTDBConnectionException) cause; + } else { + if (errMsgBuilder.length() > 0) { + errMsgBuilder.append(";"); + } + errMsgBuilder.append(cause.getMessage()); + } + } + } + if (errMsgBuilder.length() > 0) { + throw new StatementExecutionException(errMsgBuilder.toString()); + } + } + /** * insert the aligned timeseries data of a device. For each timestamp, the number of measurements * is the same. @@ -3863,6 +4056,9 @@ private void insertByGroup( if (cause instanceof IoTDBConnectionException) { throw (IoTDBConnectionException) cause; } else { + if (errMsgBuilder.length() > 0) { + errMsgBuilder.append(";"); + } errMsgBuilder.append(cause.getMessage()); } } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java index 9a59e6dc50fc..d9871d4b4e27 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/SessionConnection.java @@ -114,11 +114,11 @@ public class SessionConnection { private int timeFactor = 1_000; // TestOnly - public SessionConnection() { + public SessionConnection(String sqlDialect) { availableNodes = Collections::emptyList; this.maxRetryCount = Math.max(0, SessionConfig.MAX_RETRY_COUNT); this.retryIntervalInMs = Math.max(0, SessionConfig.RETRY_INTERVAL_IN_MS); - this.sqlDialect = "tree"; + this.sqlDialect = sqlDialect; database = null; } diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java index 5ef9f7616f62..3edfc3961736 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionPool.java @@ -41,6 +41,7 @@ import org.apache.thrift.TException; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.write.record.Tablet; @@ -127,6 +128,7 @@ public class SessionPool implements ISessionPool { private boolean enableQueryRedirection = false; private Map deviceIdToEndpoint; + private Map tableModelDeviceIdToEndpoint; private int thriftDefaultBufferSize; private int thriftMaxFrameSize; @@ -391,6 +393,7 @@ public SessionPool( this.enableRedirection = enableRedirection; if (this.enableRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); } this.connectionTimeoutInMs = connectionTimeoutInMs; this.version = version; @@ -432,6 +435,7 @@ public SessionPool( this.enableRedirection = enableRedirection; if (this.enableRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); } this.connectionTimeoutInMs = connectionTimeoutInMs; this.version = version; @@ -476,6 +480,7 @@ public SessionPool( this.enableRedirection = enableRedirection; if (this.enableRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); } this.connectionTimeoutInMs = connectionTimeoutInMs; this.version = version; @@ -497,6 +502,7 @@ public SessionPool(Builder builder) { this.enableRedirection = builder.enableRedirection; if (this.enableRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); } this.enableRecordsAutoConvertTablet = builder.enableRecordsAutoConvertTablet; this.connectionTimeoutInMs = builder.connectionTimeoutInMs; @@ -703,7 +709,13 @@ private ISession getSession() throws IoTDBConnectionException { session = constructNewSession(); try { - session.open(enableCompression, connectionTimeoutInMs, deviceIdToEndpoint, availableNodes); + + session.open( + enableCompression, + connectionTimeoutInMs, + deviceIdToEndpoint, + tableModelDeviceIdToEndpoint, + availableNodes); // avoid someone has called close() the session pool synchronized (this) { if (closed) { @@ -813,7 +825,12 @@ public void closeResultSet(SessionDataSetWrapper wrapper) { private void tryConstructNewSession() { Session session = constructNewSession(); try { - session.open(enableCompression, connectionTimeoutInMs, deviceIdToEndpoint, availableNodes); + session.open( + enableCompression, + connectionTimeoutInMs, + deviceIdToEndpoint, + tableModelDeviceIdToEndpoint, + availableNodes); // avoid someone has called close() the session pool synchronized (this) { if (closed) { @@ -3440,6 +3457,7 @@ public void setEnableRedirection(boolean enableRedirection) { this.enableRedirection = enableRedirection; if (this.enableRedirection) { deviceIdToEndpoint = new ConcurrentHashMap<>(); + tableModelDeviceIdToEndpoint = new ConcurrentHashMap<>(); } for (ISession session : queue) { session.setEnableRedirection(enableRedirection); diff --git a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionWrapper.java b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionWrapper.java index deb43f56e807..5a007921a892 100644 --- a/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionWrapper.java +++ b/iotdb-client/session/src/main/java/org/apache/iotdb/session/pool/SessionWrapper.java @@ -37,9 +37,11 @@ import java.util.concurrent.atomic.AtomicBoolean; /** - * used for SessionPool.getSession need to do some other things like calling + * NOTICE: SessionWrapper is specific to the table model. + * + *

used for SessionPool.getSession need to do some other things like calling * cleanSessionAndMayThrowConnectionException in SessionPool while encountering connection exception - * only need to putBack to SessionPool while closing + * only need to putBack to SessionPool while closing. */ public class SessionWrapper implements IPooledSession { @@ -144,7 +146,7 @@ public String getTimestampPrecision() throws TException { public void insertTablet(Tablet tablet) throws StatementExecutionException, IoTDBConnectionException { try { - session.insertTablet(tablet); + session.insertRelationalTablet(tablet); } catch (IoTDBConnectionException e) { sessionPool.cleanSessionAndMayThrowConnectionException(session); closed.set(true); diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java index 6a7a9f83051b..c83cf8c4ea47 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionCacheLeaderTest.java @@ -33,7 +33,10 @@ import org.apache.iotdb.service.rpc.thrift.TSInsertTabletsReq; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.record.Tablet.ColumnType; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Assert; @@ -56,7 +59,7 @@ public class SessionCacheLeaderTest { new ArrayList() { { add(new TEndPoint("127.0.0.1", 55560)); // default endpoint - add(new TEndPoint("127.0.0.1", 55561)); // meta leader endpoint + add(new TEndPoint("127.0.0.1", 55561)); add(new TEndPoint("127.0.0.1", 55562)); add(new TEndPoint("127.0.0.1", 55563)); } @@ -79,6 +82,20 @@ public static TEndPoint getDeviceIdBelongedEndpoint(String deviceId) { return endpoints.get(deviceId.hashCode() % endpoints.size()); } + public static TEndPoint getDeviceIdBelongedEndpoint(IDeviceID deviceId) { + if (deviceId.equals(new StringArrayDeviceID("table1", "id0"))) { + return endpoints.get(0); + } else if (deviceId.equals(new StringArrayDeviceID("table1", "id1"))) { + return endpoints.get(1); + } else if (deviceId.equals(new StringArrayDeviceID("table1", "id2"))) { + return endpoints.get(2); + } else if (deviceId.equals(new StringArrayDeviceID("table1", "id3"))) { + return endpoints.get(3); + } + + return endpoints.get(deviceId.hashCode() % endpoints.size()); + } + @Test public void testInsertRecord() throws IoTDBConnectionException, StatementExecutionException { // without leader cache @@ -607,6 +624,83 @@ public void testInsertTablets() throws IoTDBConnectionException, StatementExecut session.close(); } + @Test + public void testInsertRelationalTablet() + throws IoTDBConnectionException, StatementExecutionException { + // without leader cache + session = new MockSession("127.0.0.1", 55560, false, "table"); + session.open(); + assertNull(session.tableModelDeviceIdToEndpoint); + assertNull(session.endPointToSessionConnection); + + String tableName = "table1"; + List schemaList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("id", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); + columnTypeList.add(ColumnType.ID); + columnTypeList.add(ColumnType.MEASUREMENT); + columnTypeList.add(ColumnType.MEASUREMENT); + Tablet tablet = new Tablet(tableName, schemaList, columnTypeList, 50); + long timestamp = System.currentTimeMillis(); + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, "id" + (rowIndex % 4)); + for (int s = 1; s < 3; s++) { + long value = new Random().nextLong(); + tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + } + + if (tablet.rowSize == tablet.getMaxRowNumber()) { + session.insertRelationalTablet(tablet); + tablet.reset(); + } + timestamp++; + } + + if (tablet.rowSize != 0) { + session.insertRelationalTablet(tablet); + tablet.reset(); + } + + assertNull(session.tableModelDeviceIdToEndpoint); + assertNull(session.endPointToSessionConnection); + session.close(); + + // with leader cache + session = new MockSession("127.0.0.1", 55560, true, "table"); + session.open(); + assertEquals(0, session.tableModelDeviceIdToEndpoint.size()); + assertEquals(1, session.endPointToSessionConnection.size()); + + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, "id" + (rowIndex % 4)); + for (int s = 1; s < 3; s++) { + long value = new Random().nextLong(); + tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + } + + if (tablet.rowSize == tablet.getMaxRowNumber()) { + session.insertRelationalTablet(tablet, true); + tablet.reset(); + } + timestamp++; + } + + if (tablet.rowSize != 0) { + session.insertRelationalTablet(tablet); + tablet.reset(); + } + + assertEquals(4, session.tableModelDeviceIdToEndpoint.size()); + assertEquals(4, session.endPointToSessionConnection.size()); + session.close(); + } + @Test public void testInsertRecordsWithSessionBroken() throws StatementExecutionException { // without leader cache @@ -658,6 +752,7 @@ public void testInsertRecordsWithSessionBroken() throws StatementExecutionExcept if (time != 0 && time % 100 == 0) { try { session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); + Assert.fail(); } catch (IoTDBConnectionException e) { Assert.assertEquals( "the session connection = TEndPoint(ip:127.0.0.1, port:55560) is broken", @@ -672,6 +767,7 @@ public void testInsertRecordsWithSessionBroken() throws StatementExecutionExcept try { session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); + fail(); } catch (IoTDBConnectionException e) { Assert.assertEquals( "the session connection = TEndPoint(ip:127.0.0.1, port:55560) is broken", e.getMessage()); @@ -686,8 +782,7 @@ public void testInsertRecordsWithSessionBroken() throws StatementExecutionExcept try { session.close(); } catch (IoTDBConnectionException e) { - Assert.assertEquals( - "the session connection = TEndPoint(ip:127.0.0.1, port:55560) is broken", e.getMessage()); + Assert.fail(e.getMessage()); } // with leader cache @@ -737,8 +832,7 @@ public void testInsertRecordsWithSessionBroken() throws StatementExecutionExcept try { session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList); } catch (IoTDBConnectionException e) { - Assert.assertEquals( - "the session connection = TEndPoint(ip:127.0.0.1, port:55561) is broken", e.getMessage()); + Assert.fail(e.getMessage()); } assertEquals(3, session.deviceIdToEndpoint.size()); for (Map.Entry endPointMap : session.deviceIdToEndpoint.entrySet()) { @@ -806,6 +900,7 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept if (tablet1.rowSize == tablet1.getMaxRowNumber()) { try { session.insertTablets(tabletMap, true); + fail(); } catch (IoTDBConnectionException e) { assertEquals( "the session connection = TEndPoint(ip:127.0.0.1, port:55560) is broken", @@ -818,18 +913,6 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept timestamp++; } - if (tablet1.rowSize != 0) { - try { - session.insertTablets(tabletMap, true); - } catch (IoTDBConnectionException e) { - Assert.fail(e.getMessage()); - } - - tablet1.reset(); - tablet2.reset(); - tablet3.reset(); - } - assertNull(session.deviceIdToEndpoint); assertNull(session.endPointToSessionConnection); try { @@ -912,8 +995,7 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept try { session.insertTablets(tabletMap, true); } catch (IoTDBConnectionException e) { - Assert.assertEquals( - "the session connection = TEndPoint(ip:127.0.0.1, port:55562) is broken", e.getMessage()); + Assert.fail(e.getMessage()); } tablet1.reset(); tablet2.reset(); @@ -931,6 +1013,136 @@ public void testInsertTabletsWithSessionBroken() throws StatementExecutionExcept } } + @Test + public void testInsertRelationalTabletWithSessionBroken() throws StatementExecutionException { + // without leader cache + session = new MockSession("127.0.0.1", 55560, false, "table"); + try { + session.open(); + } catch (IoTDBConnectionException e) { + Assert.fail(e.getMessage()); + } + assertNull(session.tableModelDeviceIdToEndpoint); + assertNull(session.endPointToSessionConnection); + + // set the session connection as broken + ((MockSession) session).getLastConstructedSessionConnection().setConnectionBroken(true); + + String tableName = "table1"; + List schemaList = new ArrayList<>(); + List columnTypeList = new ArrayList<>(); + schemaList.add(new MeasurementSchema("id", TSDataType.STRING)); + schemaList.add(new MeasurementSchema("s1", TSDataType.INT64)); + schemaList.add(new MeasurementSchema("s2", TSDataType.INT64)); + columnTypeList.add(ColumnType.ID); + columnTypeList.add(ColumnType.MEASUREMENT); + columnTypeList.add(ColumnType.MEASUREMENT); + Tablet tablet = new Tablet(tableName, schemaList, columnTypeList, 50); + long timestamp = System.currentTimeMillis(); + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, "id" + (rowIndex % 4)); + for (int s = 1; s < 3; s++) { + long value = new Random().nextLong(); + tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + } + + if (tablet.rowSize == tablet.getMaxRowNumber()) { + try { + session.insertRelationalTablet(tablet); + fail(); + } catch (IoTDBConnectionException e) { + assertEquals( + "the session connection = TEndPoint(ip:127.0.0.1, port:55560) is broken", + e.getMessage()); + } + tablet.reset(); + } + timestamp++; + } + + assertNull(session.tableModelDeviceIdToEndpoint); + assertNull(session.endPointToSessionConnection); + try { + session.close(); + } catch (IoTDBConnectionException e) { + Assert.fail(e.getMessage()); + } + + // with leader cache + // rest the session connection + session = new MockSession("127.0.0.1", 55560, true, "table"); + try { + session.open(); + } catch (IoTDBConnectionException e) { + Assert.fail(e.getMessage()); + } + assertEquals(0, session.tableModelDeviceIdToEndpoint.size()); + assertEquals(1, session.endPointToSessionConnection.size()); + + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, "id" + (rowIndex % 4)); + for (int s = 1; s < 3; s++) { + long value = new Random().nextLong(); + tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + } + + if (tablet.rowSize == tablet.getMaxRowNumber()) { + try { + session.insertRelationalTablet(tablet); + } catch (IoTDBConnectionException e) { + Assert.fail(e.getMessage()); + } + tablet.reset(); + } + timestamp++; + } + + // set the session connection as broken + ((MockSession) session).getLastConstructedSessionConnection().setConnectionBroken(true); + // set connection as broken, due to we enable the cache leader, when we called + // ((MockSession) session).getLastConstructedSessionConnection(), the session's endpoint has + // been changed to EndPoint(ip:127.0.0.1, port:55562) + Assert.assertEquals( + "MockSessionConnection{ endPoint=TEndPoint(ip:127.0.0.1, port:55562)}", + ((MockSession) session).getLastConstructedSessionConnection().toString()); + + for (long row = 0; row < 100; row++) { + int rowIndex = tablet.rowSize++; + tablet.addTimestamp(rowIndex, timestamp); + tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, "id" + (rowIndex % 4)); + for (int s = 1; s < 3; s++) { + long value = new Random().nextLong(); + tablet.addValue(schemaList.get(s).getMeasurementId(), rowIndex, value); + } + + if (tablet.rowSize == tablet.getMaxRowNumber()) { + try { + session.insertRelationalTablet(tablet); + } catch (IoTDBConnectionException e) { + Assert.fail(e.getMessage()); + } + tablet.reset(); + } + timestamp++; + } + + assertEquals(3, session.tableModelDeviceIdToEndpoint.size()); + for (Map.Entry endPointEntry : + session.tableModelDeviceIdToEndpoint.entrySet()) { + assertEquals(getDeviceIdBelongedEndpoint(endPointEntry.getKey()), endPointEntry.getValue()); + } + assertEquals(3, session.endPointToSessionConnection.size()); + try { + session.close(); + } catch (IoTDBConnectionException e) { + Assert.fail(e.getMessage()); + } + } + private void addLine( List times, List> measurements, @@ -977,6 +1189,12 @@ public MockSession(String host, int rpcPort, boolean enableRedirection) { this.enableAutoFetch = false; } + public MockSession(String host, int rpcPort, boolean enableRedirection, String sqlDialect) { + this(host, rpcPort, enableRedirection); + this.sqlDialect = sqlDialect; + this.enableAutoFetch = false; + } + @Override public SessionConnection constructSessionConnection( Session session, TEndPoint endpoint, ZoneId zoneId) { @@ -996,7 +1214,7 @@ static class MockSessionConnection extends SessionConnection { private IoTDBConnectionException ioTDBConnectionException; public MockSessionConnection(Session session, TEndPoint endPoint, ZoneId zoneId) { - super(); + super(session.sqlDialect); this.endPoint = endPoint; ioTDBConnectionException = new IoTDBConnectionException( @@ -1057,7 +1275,18 @@ protected void insertTablet(TSInsertTabletReq request) if (isConnectionBroken()) { throw ioTDBConnectionException; } - throw new RedirectException(getDeviceIdBelongedEndpoint(request.prefixPath)); + if (request.writeToTable) { + if (request.size >= 50) { + // multi devices + List endPoints = new ArrayList<>(); + for (int i = 0; i < request.size; i++) { + endPoints.add(endpoints.get(i % 4)); + } + throw new RedirectException(endPoints); + } + } else { + throw new RedirectException(getDeviceIdBelongedEndpoint(request.prefixPath)); + } } @Override diff --git a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java index 0785bc800d97..a10d9b1d6f4d 100644 --- a/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java +++ b/iotdb-client/session/src/test/java/org/apache/iotdb/session/SessionConnectionTest.java @@ -84,7 +84,7 @@ public class SessionConnectionTest { @Before public void setUp() throws IoTDBConnectionException, StatementExecutionException, TException { MockitoAnnotations.initMocks(this); - sessionConnection = new SessionConnection(); + sessionConnection = new SessionConnection("tree"); Whitebox.setInternalState(sessionConnection, "transport", transport); Whitebox.setInternalState(sessionConnection, "client", client); session = diff --git a/iotdb-core/ainode/pyproject.toml b/iotdb-core/ainode/pyproject.toml index 6211d2301365..c7f773b05908 100644 --- a/iotdb-core/ainode/pyproject.toml +++ b/iotdb-core/ainode/pyproject.toml @@ -21,7 +21,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "apache-iotdb-ainode" -version = "1.4.0.dev" +version = "2.0.0.dev" description = "Apache IoTDB AINode" readme = "README.md" authors = ["Apache Software Foundation "] diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java index ea801a26632b..52f8226351f3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java @@ -69,6 +69,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TDropTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TFetchSchemaBlackListReq; import org.apache.iotdb.mpp.rpc.thrift.TInactiveTriggerInstanceReq; +import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateMatchedSchemaCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq; import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaReq; @@ -261,6 +262,11 @@ protected void initActionMapBuilder() { (req, client, handler) -> client.fetchSchemaBlackList( (TFetchSchemaBlackListReq) req, (FetchSchemaBlackListRPCHandler) handler)); + actionMapBuilder.put( + CnToDnRequestType.INVALIDATE_SCHEMA_CACHE, + (req, client, handler) -> + client.invalidateSchemaCache( + (TInvalidateCacheReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( CnToDnRequestType.INVALIDATE_MATCHED_SCHEMA_CACHE, (req, client, handler) -> diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java index d8cfd1a2f912..ac4d05cd39e9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/rpc/DataNodeAsyncRequestRPCHandler.java @@ -202,6 +202,7 @@ public static DataNodeAsyncRequestRPCHandler buildHandler( case LOAD_CONFIGURATION: case SET_SYSTEM_STATUS: case UPDATE_REGION_ROUTE_MAP: + case INVALIDATE_SCHEMA_CACHE: case INVALIDATE_MATCHED_SCHEMA_CACHE: case UPDATE_TEMPLATE: case UPDATE_TABLE: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 86bc4e579ed8..78935778b66d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -159,7 +159,7 @@ public class ConfigNodeConfig { systemDir + File.separator + "pipe" + File.separator + "receiver"; /** Procedure Evict ttl. */ - private int procedureCompletedEvictTTL = 800; + private int procedureCompletedEvictTTL = 60; /** Procedure completed clean interval. */ private int procedureCompletedCleanInterval = 30; @@ -308,14 +308,10 @@ private void formulateFolders() { pipeReceiverFileDir = addHomeDir(pipeReceiverFileDir); } - private String addHomeDir(String dir) { - String homeDir = System.getProperty(ConfigNodeConstant.CONFIGNODE_HOME, null); - if (!new File(dir).isAbsolute() && homeDir != null && homeDir.length() > 0) { - if (!homeDir.endsWith(File.separator)) { - dir = homeDir + File.separatorChar + dir; - } else { - dir = homeDir + dir; - } + public static String addHomeDir(String dir) { + final String homeDir = System.getProperty(ConfigNodeConstant.CONFIGNODE_HOME, null); + if (!new File(dir).isAbsolute() && homeDir != null && !homeDir.isEmpty()) { + dir = !homeDir.endsWith(File.separator) ? homeDir + File.separatorChar + dir : homeDir + dir; } return dir; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index 7bfa5c50c69d..9d020aa52d77 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -20,7 +20,13 @@ package org.apache.iotdb.confignode.consensus.request; import org.apache.iotdb.commons.exception.runtime.SerializationRunTimeException; +import org.apache.iotdb.confignode.consensus.request.read.ainode.GetAINodeConfigurationPlan; +import org.apache.iotdb.confignode.consensus.request.read.model.GetModelInfoPlan; +import org.apache.iotdb.confignode.consensus.request.read.model.ShowModelPlan; import org.apache.iotdb.confignode.consensus.request.read.subscription.ShowTopicPlan; +import org.apache.iotdb.confignode.consensus.request.write.ainode.RegisterAINodePlan; +import org.apache.iotdb.confignode.consensus.request.write.ainode.RemoveAINodePlan; +import org.apache.iotdb.confignode.consensus.request.write.ainode.UpdateAINodePlan; import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan; import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan; @@ -43,6 +49,10 @@ import org.apache.iotdb.confignode.consensus.request.write.datanode.UpdateDataNodePlan; import org.apache.iotdb.confignode.consensus.request.write.function.CreateFunctionPlan; import org.apache.iotdb.confignode.consensus.request.write.function.DropFunctionPlan; +import org.apache.iotdb.confignode.consensus.request.write.model.CreateModelPlan; +import org.apache.iotdb.confignode.consensus.request.write.model.DropModelInNodePlan; +import org.apache.iotdb.confignode.consensus.request.write.model.DropModelPlan; +import org.apache.iotdb.confignode.consensus.request.write.model.UpdateModelInfoPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.AddRegionLocationPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.CreateDataPartitionPlan; import org.apache.iotdb.confignode.consensus.request.write.partition.CreateSchemaPartitionPlan; @@ -167,6 +177,18 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept case RemoveDataNode: plan = new RemoveDataNodePlan(); break; + case RegisterAINode: + plan = new RegisterAINodePlan(); + break; + case RemoveAINode: + plan = new RemoveAINodePlan(); + break; + case GetAINodeConfiguration: + plan = new GetAINodeConfigurationPlan(); + break; + case UpdateAINodeConfiguration: + plan = new UpdateAINodePlan(); + break; case CreateDatabase: plan = new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase); break; @@ -420,6 +442,24 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept case UPDATE_CQ_LAST_EXEC_TIME: plan = new UpdateCQLastExecTimePlan(); break; + case CreateModel: + plan = new CreateModelPlan(); + break; + case UpdateModelInfo: + plan = new UpdateModelInfoPlan(); + break; + case DropModel: + plan = new DropModelPlan(); + break; + case ShowModel: + plan = new ShowModelPlan(); + break; + case DropModelInNode: + plan = new DropModelInNodePlan(); + break; + case GetModelInfo: + plan = new GetModelInfoPlan(); + break; case CreatePipePlugin: plan = new CreatePipePluginPlan(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 3b011e5cd838..5d5006757643 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -161,6 +161,7 @@ public enum ConfigPhysicalPlanType { AddTableColumn((short) 853), SetTableProperties((short) 854), ShowTable((short) 855), + FetchTable((short) 856), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ConfigPhysicalReadPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ConfigPhysicalReadPlan.java index b8652f03a1b5..032c21a731c0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ConfigPhysicalReadPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ConfigPhysicalReadPlan.java @@ -34,11 +34,11 @@ protected ConfigPhysicalReadPlan(final ConfigPhysicalPlanType type) { @Override protected void serializeImpl(final DataOutputStream stream) throws IOException { - throw new UnsupportedOperationException("Read request does not need to be serialized."); + // Read request does not need to be serialized } @Override protected void deserializeImpl(final ByteBuffer buffer) throws IOException { - throw new UnsupportedOperationException("Read request does not need to be deserialized."); + // Read request does not need to be deserialized } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ainode/GetAINodeConfigurationPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ainode/GetAINodeConfigurationPlan.java index b080cf77c5dd..7222a8f53f8c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ainode/GetAINodeConfigurationPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/ainode/GetAINodeConfigurationPlan.java @@ -22,10 +22,18 @@ import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + public class GetAINodeConfigurationPlan extends ConfigPhysicalReadPlan { // if aiNodeId is set to -1, return all AINode configurations. - private final int aiNodeId; + private int aiNodeId; + + public GetAINodeConfigurationPlan() { + super(ConfigPhysicalPlanType.GetAINodeConfiguration); + } public GetAINodeConfigurationPlan(final int aiNodeId) { super(ConfigPhysicalPlanType.GetAINodeConfiguration); @@ -36,6 +44,17 @@ public int getAiNodeId() { return aiNodeId; } + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + stream.writeInt(aiNodeId); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.aiNodeId = buffer.getInt(); + } + @Override public boolean equals(final Object o) { if (this == o) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java index dfec065f2069..9c33c2678829 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/GetModelInfoPlan.java @@ -23,11 +23,20 @@ import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; import org.apache.iotdb.confignode.rpc.thrift.TGetModelInfoReq; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Objects; public class GetModelInfoPlan extends ConfigPhysicalReadPlan { - private final String modelId; + private String modelId; + + public GetModelInfoPlan() { + super(ConfigPhysicalPlanType.GetModelInfo); + } public GetModelInfoPlan(final TGetModelInfoReq getModelInfoReq) { super(ConfigPhysicalPlanType.GetModelInfo); @@ -38,6 +47,17 @@ public String getModelId() { return modelId; } + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(modelId, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + this.modelId = ReadWriteIOUtils.readString(buffer); + } + @Override public boolean equals(final Object o) { if (this == o) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java index c3d0ff79a37a..df924c97f5b7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/model/ShowModelPlan.java @@ -23,12 +23,21 @@ import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; import org.apache.iotdb.confignode.rpc.thrift.TShowModelReq; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Objects; public class ShowModelPlan extends ConfigPhysicalReadPlan { private String modelName; + public ShowModelPlan() { + super(ConfigPhysicalPlanType.ShowModel); + } + public ShowModelPlan(final TShowModelReq showModelReq) { super(ConfigPhysicalPlanType.ShowModel); if (showModelReq.isSetModelId()) { @@ -44,6 +53,21 @@ public String getModelName() { return modelName; } + @Override + protected void serializeImpl(DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + ReadWriteIOUtils.write(modelName != null, stream); + ReadWriteIOUtils.write(modelName, stream); + } + + @Override + protected void deserializeImpl(ByteBuffer buffer) throws IOException { + boolean isSetModelId = ReadWriteIOUtils.readBool(buffer); + if (isSetModelId) { + this.modelName = ReadWriteIOUtils.readString(buffer); + } + } + @Override public boolean equals(final Object o) { if (this == o) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/FetchTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/FetchTablePlan.java new file mode 100644 index 000000000000..a69eda99d79a --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/FetchTablePlan.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.read.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; +import org.apache.iotdb.confignode.consensus.request.read.ConfigPhysicalReadPlan; + +import java.util.Map; +import java.util.Set; + +public class FetchTablePlan extends ConfigPhysicalReadPlan { + + private final Map> fetchTableMap; + + public FetchTablePlan(final Map> fetchTableMap) { + super(ConfigPhysicalPlanType.FetchTable); + this.fetchTableMap = fetchTableMap; + } + + public Map> getFetchTableMap() { + return fetchTableMap; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/FetchTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/FetchTableResp.java new file mode 100644 index 000000000000..6985de613fb0 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/table/FetchTableResp.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.response.table; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.confignode.rpc.thrift.TFetchTableResp; +import org.apache.iotdb.consensus.common.DataSet; + +import java.util.Map; + +public class FetchTableResp implements DataSet { + private final TSStatus status; + private final Map> fetchTableMap; + + public FetchTableResp( + final TSStatus status, final Map> fetchTableMap) { + this.status = status; + this.fetchTableMap = fetchTableMap; + } + + public TFetchTableResp convertToTFetchTableResp() { + return new TFetchTableResp(status) + .setTableInfoMap(TsTableInternalRPCUtil.serializeTableFetchResult(fetchTableMap)); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 552c2d44dceb..87dd13e654c4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -166,6 +166,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; +import org.apache.iotdb.confignode.rpc.thrift.TFetchTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllTemplatesResp; @@ -2582,6 +2583,28 @@ public TShowTableResp showTables(final String database) { : new TShowTableResp(status); } + @Override + public TFetchTableResp fetchTables(final Map> fetchTableMap) { + final TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? clusterSchemaManager.fetchTables( + fetchTableMap.entrySet().stream() + .filter( + entry -> { + entry + .getValue() + .removeIf( + table -> + procedureManager + .checkDuplicateTableTask( + entry.getKey(), null, table, null, null) + .getRight()); + return !entry.getValue().isEmpty(); + }) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))) + : new TFetchTableResp(status); + } + @Override public DataSet registerAINode(TAINodeRegisterReq req) { TSStatus status = confirmLeader(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 013160923cb4..640c82d6653f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -91,6 +91,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; +import org.apache.iotdb.confignode.rpc.thrift.TFetchTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllTemplatesResp; @@ -148,6 +149,7 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.Map; +import java.util.Set; /** * A subset of services provided by {@link ConfigManager}. For use internally only, passed to @@ -833,4 +835,6 @@ TDataPartitionTableResp getOrCreateDataPartition( TSStatus alterTable(final TAlterTableReq req); TShowTableResp showTables(final String database); + + TFetchTableResp fetchTables(final Map> fetchTableMap); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 889e56f15984..bfc1d7809e35 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -234,7 +234,7 @@ public TSStatus deleteDatabases( while (executor.isRunning() && System.currentTimeMillis() - startCheckTimeForProcedures < PROCEDURE_WAIT_TIME_OUT) { final Pair procedureIdDuplicatePair = - awaitDuplicateTableTask( + checkDuplicateTableTask( database, null, null, null, ProcedureType.CREATE_TABLE_PROCEDURE); hasOverlappedTask = procedureIdDuplicatePair.getRight(); @@ -1520,7 +1520,7 @@ private TSStatus executeWithoutDuplicate( long procedureId; synchronized (this) { final Pair procedureIdDuplicatePair = - awaitDuplicateTableTask(database, table, tableName, queryId, thisType); + checkDuplicateTableTask(database, table, tableName, queryId, thisType); procedureId = procedureIdDuplicatePair.getLeft(); if (procedureId == -1) { @@ -1533,16 +1533,12 @@ private TSStatus executeWithoutDuplicate( } } final List procedureStatus = new ArrayList<>(); - final boolean isSucceed = - waitingProcedureFinished(Collections.singletonList(procedureId), procedureStatus); - if (isSucceed) { - return StatusUtils.OK; - } else { - return procedureStatus.get(0); - } + return waitingProcedureFinished(Collections.singletonList(procedureId), procedureStatus) + ? StatusUtils.OK + : procedureStatus.get(0); } - private Pair awaitDuplicateTableTask( + public Pair checkDuplicateTableTask( final String database, final TsTable table, final String tableName, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java index 13d58397c9f1..a5d4e3d01669 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java @@ -47,6 +47,7 @@ import org.apache.iotdb.confignode.manager.node.NodeManager; import org.apache.iotdb.confignode.manager.partition.PartitionManager; import org.apache.iotdb.consensus.ConsensusFactory; +import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeReq; import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeResp; import org.apache.iotdb.mpp.rpc.thrift.TRegionRouteReq; @@ -59,6 +60,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -242,7 +244,40 @@ private void balanceRegionLeader( } } } + getLoadManager().forceUpdateConsensusGroupCache(successTransferMap); + + invalidateSchemaCacheOfOldLeaders(currentLeaderMap, successTransferMap.keySet()); + } + + private void invalidateSchemaCacheOfOldLeaders( + Map oldLeaderMap, Set successTransferSet) { + DataNodeAsyncRequestContext invalidateSchemaCacheRequestHandler = + new DataNodeAsyncRequestContext<>(CnToDnRequestType.INVALIDATE_SCHEMA_CACHE); + AtomicInteger requestIndex = new AtomicInteger(0); + oldLeaderMap.entrySet().stream() + .filter(entry -> TConsensusGroupType.DataRegion == entry.getKey().getType()) + .filter(entry -> successTransferSet.contains(entry.getKey())) + .forEach( + entry -> { + // set target + Integer dataNodeId = entry.getValue(); + TDataNodeLocation dataNodeLocation = + getNodeManager().getRegisteredDataNode(dataNodeId).getLocation(); + if (dataNodeLocation == null) { + LOGGER.warn("DataNodeLocation is null, datanodeId {}", dataNodeId); + return; + } + invalidateSchemaCacheRequestHandler.putNodeLocation( + requestIndex.getAndIncrement(), dataNodeLocation); + // set req + TConsensusGroupId consensusGroupId = entry.getKey(); + String database = getPartitionManager().getRegionStorageGroup(consensusGroupId); + invalidateSchemaCacheRequestHandler.putRequest( + requestIndex.get(), new TInvalidateCacheReq(true, database)); + }); + CnToDnInternalServiceAsyncRequestManager.getInstance() + .sendAsyncRequest(invalidateSchemaCacheRequestHandler); } public synchronized void balanceRegionLeaderAndPriority() { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/runtime/PipeConfigNodeRuntimeAgent.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/runtime/PipeConfigNodeRuntimeAgent.java index c3262c681818..cc6056522ef8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/runtime/PipeConfigNodeRuntimeAgent.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/runtime/PipeConfigNodeRuntimeAgent.java @@ -35,7 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; public class PipeConfigNodeRuntimeAgent implements IService { @@ -90,12 +89,13 @@ public ConfigRegionListeningQueue listener() { return regionListener.listener(); } - public void increaseListenerReference(PipeParameters parameters) throws IllegalPathException { + public void increaseListenerReference(final PipeParameters parameters) + throws IllegalPathException { regionListener.increaseReference(parameters); } - public void decreaseListenerReference(PipeParameters parameters) - throws IllegalPathException, IOException { + public void decreaseListenerReference(final PipeParameters parameters) + throws IllegalPathException { regionListener.decreaseReference(parameters); } @@ -120,7 +120,7 @@ public boolean isLeaderReady() { //////////////////////////// Runtime Exception Handlers //////////////////////////// - public void report(EnrichedEvent event, PipeRuntimeException pipeRuntimeException) { + public void report(final EnrichedEvent event, final PipeRuntimeException pipeRuntimeException) { if (event.getPipeTaskMeta() != null) { report(event.getPipeTaskMeta(), pipeRuntimeException); } else { @@ -128,7 +128,8 @@ public void report(EnrichedEvent event, PipeRuntimeException pipeRuntimeExceptio } } - private void report(PipeTaskMeta pipeTaskMeta, PipeRuntimeException pipeRuntimeException) { + private void report( + final PipeTaskMeta pipeTaskMeta, final PipeRuntimeException pipeRuntimeException) { LOGGER.warn( "Report PipeRuntimeException to local PipeTaskMeta({}), exception message: {}", pipeTaskMeta, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/runtime/PipeConfigRegionListener.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/runtime/PipeConfigRegionListener.java index 458ed1c57a82..7c8bcca9f366 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/runtime/PipeConfigRegionListener.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/agent/runtime/PipeConfigRegionListener.java @@ -24,7 +24,6 @@ import org.apache.iotdb.confignode.manager.pipe.extractor.ConfigRegionListeningQueue; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; -import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; public class PipeConfigRegionListener { @@ -38,7 +37,7 @@ public synchronized ConfigRegionListeningQueue listener() { return listeningQueue; } - public synchronized void increaseReference(PipeParameters parameters) + public synchronized void increaseReference(final PipeParameters parameters) throws IllegalPathException { if (!ConfigRegionListeningFilter.parseListeningPlanTypeSet(parameters).isEmpty()) { listeningQueueReferenceCount++; @@ -48,8 +47,8 @@ public synchronized void increaseReference(PipeParameters parameters) } } - public synchronized void decreaseReference(PipeParameters parameters) - throws IllegalPathException, IOException { + public synchronized void decreaseReference(final PipeParameters parameters) + throws IllegalPathException { if (!ConfigRegionListeningFilter.parseListeningPlanTypeSet(parameters).isEmpty()) { listeningQueueReferenceCount--; if (listeningQueueReferenceCount == 0) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java index 3e4ca8ac9c50..307151df706b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/connector/protocol/IoTDBConfigRegionConnector.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.pipe.connector.client.IoTDBSyncClientManager; import org.apache.iotdb.commons.pipe.connector.payload.thrift.request.PipeTransferFilePieceReq; import org.apache.iotdb.commons.pipe.connector.protocol.IoTDBSslSyncConnector; +import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.manager.pipe.connector.client.IoTDBConfigNodeSyncClientManager; import org.apache.iotdb.confignode.manager.pipe.connector.payload.PipeTransferConfigPlanReq; import org.apache.iotdb.confignode.manager.pipe.connector.payload.PipeTransferConfigSnapshotPieceReq; @@ -68,7 +69,7 @@ protected IoTDBSyncClientManager constructClient( return new IoTDBConfigNodeSyncClientManager( nodeUrls, useSSL, - trustStorePath, + Objects.nonNull(trustStorePath) ? ConfigNodeConfig.addHomeDir(trustStorePath) : null, trustStorePwd, loadBalanceStrategy, shouldReceiverConvertOnTypeMismatch, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 15422f572ec2..1dba249dde2d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -39,6 +39,7 @@ import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabasePlan; import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan; +import org.apache.iotdb.confignode.consensus.request.read.table.FetchTablePlan; import org.apache.iotdb.confignode.consensus.request.read.table.ShowTablePlan; import org.apache.iotdb.confignode.consensus.request.read.template.GetAllSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.read.template.GetAllTemplateSetInfoPlan; @@ -63,6 +64,7 @@ import org.apache.iotdb.confignode.consensus.response.database.CountDatabaseResp; import org.apache.iotdb.confignode.consensus.response.database.DatabaseSchemaResp; import org.apache.iotdb.confignode.consensus.response.partition.PathInfoResp; +import org.apache.iotdb.confignode.consensus.response.table.FetchTableResp; import org.apache.iotdb.confignode.consensus.response.table.ShowTableResp; import org.apache.iotdb.confignode.consensus.response.template.AllTemplateSetInfoResp; import org.apache.iotdb.confignode.consensus.response.template.TemplateInfoResp; @@ -76,6 +78,7 @@ import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseInfo; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TFetchTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllTemplatesResp; import org.apache.iotdb.confignode.rpc.thrift.TGetPathsSetTemplatesResp; import org.apache.iotdb.confignode.rpc.thrift.TGetTemplateResp; @@ -1073,6 +1076,19 @@ public TShowTableResp showTables(final String database) { } } + public TFetchTableResp fetchTables(final Map> fetchTableMap) { + try { + return ((FetchTableResp) + configManager.getConsensusManager().read(new FetchTablePlan(fetchTableMap))) + .convertToTFetchTableResp(); + } catch (final ConsensusException e) { + LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); + final TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + res.setMessage(e.getMessage()); + return new TFetchTableResp(res); + } + } + public byte[] getAllTableInfoForDataNodeActivation() { return TsTableInternalRPCUtil.serializeTableInitializationInfo( clusterSchemaInfo.getAllUsingTables(), clusterSchemaInfo.getAllPreCreateTables()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index d1dc7420212a..2613ec665100 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -45,6 +45,7 @@ import org.apache.iotdb.confignode.consensus.request.read.pipe.plugin.GetPipePluginJarPlan; import org.apache.iotdb.confignode.consensus.request.read.region.GetRegionIdPlan; import org.apache.iotdb.confignode.consensus.request.read.region.GetRegionInfoListPlan; +import org.apache.iotdb.confignode.consensus.request.read.table.FetchTablePlan; import org.apache.iotdb.confignode.consensus.request.read.table.ShowTablePlan; import org.apache.iotdb.confignode.consensus.request.read.template.CheckTemplateSettablePlan; import org.apache.iotdb.confignode.consensus.request.read.template.GetPathsSetTemplatePlan; @@ -311,6 +312,8 @@ public DataSet executeQueryPlan(final ConfigPhysicalReadPlan req) return clusterSchemaInfo.getTemplateSetInfo((GetTemplateSetInfoPlan) req); case ShowTable: return clusterSchemaInfo.showTables((ShowTablePlan) req); + case FetchTable: + return clusterSchemaInfo.fetchTables((FetchTablePlan) req); case GetTriggerTable: return triggerInfo.getTriggerTable((GetTriggerTablePlan) req); case GetTriggerLocation: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java index 7dd4af334a72..54570321e9ef 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java @@ -51,12 +51,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -722,21 +723,24 @@ public void processLoadSnapshot(File snapshotDir) throws IOException, TException aiNodeInfoReadWriteLock.writeLock().lock(); versionInfoReadWriteLock.writeLock().lock(); - try (FileInputStream fileInputStream = new FileInputStream(snapshotFile); - TIOStreamTransport tioStreamTransport = new TIOStreamTransport(fileInputStream)) { + try (ByteArrayInputStream inputStream = + new ByteArrayInputStream(Files.readAllBytes(snapshotFile.toPath())); + TIOStreamTransport tioStreamTransport = new TIOStreamTransport(inputStream)) { TProtocol protocol = new TBinaryProtocol(tioStreamTransport); clear(); - nextNodeId.set(ReadWriteIOUtils.readInt(fileInputStream)); + nextNodeId.set(ReadWriteIOUtils.readInt(inputStream)); - deserializeRegisteredConfigNode(fileInputStream, protocol); + deserializeRegisteredConfigNode(inputStream, protocol); - deserializeRegisteredDataNode(fileInputStream, protocol); + deserializeRegisteredDataNode(inputStream, protocol); - deserializeRegisteredAINode(fileInputStream, protocol); + // TODO: Compatibility design. Should replace this function to actual deserialization method + // in IoTDB 2.2 / 1.5 + tryDeserializeRegisteredAINode(inputStream, protocol); - deserializeBuildInfo(fileInputStream); + deserializeBuildInfo(inputStream); } finally { versionInfoReadWriteLock.writeLock().unlock(); @@ -770,6 +774,18 @@ private void deserializeRegisteredDataNode(InputStream inputStream, TProtocol pr } } + private void tryDeserializeRegisteredAINode(ByteArrayInputStream inputStream, TProtocol protocol) + throws IOException { + try { + // 0 has no meaning here + inputStream.mark(0); + deserializeRegisteredAINode(inputStream, protocol); + } catch (IOException | TException ignore) { + // Exception happens here means that the data is upgraded from the old version + inputStream.reset(); + } + } + private void deserializeRegisteredAINode(InputStream inputStream, TProtocol protocol) throws IOException, TException { int size = ReadWriteIOUtils.readInt(inputStream); @@ -805,6 +821,7 @@ public void clear() { nextNodeId.set(-1); registeredDataNodes.clear(); registeredConfigNodes.clear(); + registeredAINodes.clear(); nodeVersionInfo.clear(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java index fff581646262..b80070c1d217 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeInfo.java @@ -159,14 +159,35 @@ public TSStatus dropPipe(final DropPipePlanV2 plan) { public TSStatus alterPipe(final AlterPipePlanV2 plan) { try { + final Optional pipeMetaBeforeAlter = + Optional.ofNullable( + pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeStaticMeta().getPipeName())); + pipeTaskInfo.alterPipe(plan); - PipeConfigNodeAgent.task() - .handleSinglePipeMetaChanges( - pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeStaticMeta().getPipeName())); - PipeTemporaryMetaMetrics.getInstance() - .handleTemporaryMetaChanges(pipeTaskInfo.getPipeMetaList()); - return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + final TPushPipeMetaRespExceptionMessage message = + PipeConfigNodeAgent.task() + .handleSinglePipeMetaChanges( + pipeTaskInfo.getPipeMetaByPipeName(plan.getPipeStaticMeta().getPipeName())); + if (message == null) { + pipeMetaBeforeAlter.ifPresent( + meta -> { + try { + PipeConfigNodeAgent.runtime() + .decreaseListenerReference(meta.getStaticMeta().getExtractorParameters()); + } catch (final Exception e) { + throw new PipeException("Failed to decrease listener reference", e); + } + }); + PipeConfigNodeAgent.runtime() + .increaseListenerReference(plan.getPipeStaticMeta().getExtractorParameters()); + PipeTemporaryMetaMetrics.getInstance() + .handleTemporaryMetaChanges(pipeTaskInfo.getPipeMetaList()); + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } else { + return new TSStatus(TSStatusCode.PIPE_ERROR.getStatusCode()) + .setMessage(message.getMessage()); + } } catch (final Exception e) { LOGGER.error("Failed to alter pipe", e); return new TSStatus(TSStatusCode.PIPE_ERROR.getStatusCode()) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 982c145b9611..dc743217f4e0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -35,6 +35,7 @@ import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.confignode.consensus.request.read.database.CountDatabasePlan; import org.apache.iotdb.confignode.consensus.request.read.database.GetDatabasePlan; +import org.apache.iotdb.confignode.consensus.request.read.table.FetchTablePlan; import org.apache.iotdb.confignode.consensus.request.read.table.ShowTablePlan; import org.apache.iotdb.confignode.consensus.request.read.template.CheckTemplateSettablePlan; import org.apache.iotdb.confignode.consensus.request.read.template.GetPathsSetTemplatePlan; @@ -63,6 +64,7 @@ import org.apache.iotdb.confignode.consensus.response.database.CountDatabaseResp; import org.apache.iotdb.confignode.consensus.response.database.DatabaseSchemaResp; import org.apache.iotdb.confignode.consensus.response.partition.PathInfoResp; +import org.apache.iotdb.confignode.consensus.response.table.FetchTableResp; import org.apache.iotdb.confignode.consensus.response.table.ShowTableResp; import org.apache.iotdb.confignode.consensus.response.template.AllTemplateSetInfoResp; import org.apache.iotdb.confignode.consensus.response.template.TemplateInfoResp; @@ -1102,6 +1104,27 @@ public ShowTableResp showTables(final ShowTablePlan plan) { } } + public FetchTableResp fetchTables(final FetchTablePlan plan) { + databaseReadWriteLock.readLock().lock(); + try { + final Map> result = new HashMap<>(); + for (final Map.Entry> database2Tables : + plan.getFetchTableMap().entrySet()) { + result.put( + database2Tables.getKey(), + mTree.getSpecificTablesUnderSpecificDatabase( + getQualifiedDatabasePartialPath(database2Tables.getKey()), + database2Tables.getValue())); + } + return new FetchTableResp(StatusUtils.OK, result); + } catch (final MetadataException e) { + return new FetchTableResp( + RpcUtils.getStatus(e.getErrorCode(), e.getMessage()), Collections.emptyMap()); + } finally { + databaseReadWriteLock.readLock().unlock(); + } + } + public Map> getAllUsingTables() { databaseReadWriteLock.readLock().lock(); try { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index c7a2af3a27f3..94659902ccf4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -685,6 +685,23 @@ public List getAllUsingTablesUnderSpecificDatabase(final PartialPath da .collect(Collectors.toList()); } + public Map getSpecificTablesUnderSpecificDatabase( + final PartialPath databasePath, final Set tables) throws MetadataException { + final IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(databasePath).getAsMNode(); + final Map result = new HashMap<>(); + tables.forEach( + table -> { + final IConfigMNode child = databaseNode.getChildren().get(table); + if (child instanceof ConfigTableNode + && ((ConfigTableNode) child).getStatus().equals(TableNodeStatus.USING)) { + result.put(table, ((ConfigTableNode) child).getTable()); + } else { + result.put(table, null); + } + }); + return result; + } + public Map> getAllUsingTables() { return getAllDatabasePaths().stream() .collect( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java index ccf3afeb4e64..cea15a1bfb68 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java @@ -31,7 +31,7 @@ /** Internal cleaner that removes the completed procedure results after a TTL. */ public class CompletedProcedureRecycler extends InternalProcedure { private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureRecycler.class); - private static final int DEFAULT_BATCH_SIZE = 32; + private static final int DEFAULT_BATCH_SIZE = 8; private final long evictTTL; private final Map> completed; private final IProcedureStore store; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index 3e77b6dad8bf..21d6e3f058d5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -167,7 +167,6 @@ private void commitRelease(final ConfigNodeProcedureEnv env) { database, table.getTableName(), failedResults); - // TODO: Handle commit failure } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index f077e251ea64..fc6944eff42b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -297,7 +297,6 @@ private void commitReleaseTable(final ConfigNodeProcedureEnv env) { database, table.getTableName(), failedResults); - // TODO: Handle commit failure } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java index 10c6d9760121..039982a99e1c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java @@ -181,7 +181,6 @@ private void commitRelease(final ConfigNodeProcedureEnv env) { database, table.getTableName(), failedResults); - // TODO: Handle commit failure } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index b509001b984c..d64e82026a40 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -133,6 +133,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; +import org.apache.iotdb.confignode.rpc.thrift.TFetchTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllTemplatesResp; @@ -214,6 +215,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; /** ConfigNodeRPCServer exposes the interface that interacts with the DataNode */ @@ -381,9 +383,10 @@ public TShowVariablesResp showVariables() { } @Override - public TSStatus setDatabase(TDatabaseSchema databaseSchema) { + public TSStatus setDatabase(final TDatabaseSchema databaseSchema) { TSStatus errorResp = null; - boolean isSystemDatabase = databaseSchema.getName().equals(SchemaConstant.SYSTEM_DATABASE); + final boolean isSystemDatabase = + databaseSchema.getName().equals(SchemaConstant.SYSTEM_DATABASE); if (databaseSchema.getTTL() < 0) { errorResp = @@ -463,9 +466,9 @@ public TSStatus setDatabase(TDatabaseSchema databaseSchema) { databaseSchema.setMaxSchemaRegionGroupNum(databaseSchema.getMinSchemaRegionGroupNum()); databaseSchema.setMaxDataRegionGroupNum(databaseSchema.getMinDataRegionGroupNum()); - DatabaseSchemaPlan setPlan = + final DatabaseSchemaPlan setPlan = new DatabaseSchemaPlan(ConfigPhysicalPlanType.CreateDatabase, databaseSchema); - TSStatus resp = configManager.setDatabase(setPlan); + final TSStatus resp = configManager.setDatabase(setPlan); // Print log to record the ConfigNode that performs the set SetDatabaseRequest LOGGER.info("Execute SetDatabase: {} with result: {}", databaseSchema, resp); @@ -1304,4 +1307,9 @@ public TSStatus alterTable(final TAlterTableReq req) { public TShowTableResp showTables(final String database) { return configManager.showTables(database); } + + @Override + public TFetchTableResp fetchTables(final Map> fetchTableMap) { + return configManager.fetchTables(fetchTableMap); + } } diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java index ed089ff63bfe..4d4e38d239ce 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java @@ -70,6 +70,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -435,6 +436,9 @@ public List getAllConsensusGroupIdsWithoutStarting() { } public static List getConsensusGroupIdsFromDir(File storageDir, Logger logger) { + if (!storageDir.exists()) { + return Collections.emptyList(); + } List consensusGroupIds = new ArrayList<>(); try (DirectoryStream stream = Files.newDirectoryStream(storageDir.toPath())) { for (Path path : stream) { diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java index 85786cc70872..25a1fb25e68d 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java @@ -390,6 +390,9 @@ public DataSet read(ConsensusGroupId groupId, IConsensusRequest request) } else { throw new RatisRequestFailedException(e); } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RatisReadUnavailableException(e); } catch (Exception e) { throw new RatisRequestFailedException(e); } @@ -773,6 +776,9 @@ public List getAllConsensusGroupIds() { @Override public List getAllConsensusGroupIdsWithoutStarting() { + if (!storageDir.exists()) { + return Collections.emptyList(); + } List consensusGroupIds = new ArrayList<>(); try (DirectoryStream stream = Files.newDirectoryStream(storageDir.toPath())) { for (Path path : stream) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index e176692361ef..1d369eeccd21 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -439,8 +439,8 @@ public class IoTDBConfig { /** Compact the unsequence files into the overlapped sequence files */ private volatile boolean enableCrossSpaceCompaction = true; - /** Enable the service for AINode */ - private boolean enableAINodeService = false; + /** Enable auto repair compaction */ + private volatile boolean enableAutoRepairCompaction = true; /** The buffer for sort operation */ private long sortBufferSize = 1024 * 1024L; @@ -920,9 +920,6 @@ public class IoTDBConfig { /** Internal port for coordinator */ private int internalPort = 10730; - /** Port for AINode */ - private int aiNodePort = 10780; - /** Internal port for dataRegion consensus protocol */ private int dataRegionConsensusPort = 10760; @@ -1045,6 +1042,8 @@ public class IoTDBConfig { /** Policy of DataNodeSchemaCache eviction */ private String dataNodeSchemaCacheEvictionPolicy = "FIFO"; + private int dataNodeTableCacheSemaphorePermitNum = 5; + private String readConsistencyLevel = "strong"; /** Maximum execution time of a DriverTask */ @@ -2871,12 +2870,12 @@ public void setEnableCrossSpaceCompaction(boolean enableCrossSpaceCompaction) { this.enableCrossSpaceCompaction = enableCrossSpaceCompaction; } - public boolean isEnableAINodeService() { - return enableAINodeService; + public boolean isEnableAutoRepairCompaction() { + return enableAutoRepairCompaction; } - public void setEnableAINodeService(boolean enableAINodeService) { - this.enableAINodeService = enableAINodeService; + public void setEnableAutoRepairCompaction(boolean enableAutoRepairCompaction) { + this.enableAutoRepairCompaction = enableAutoRepairCompaction; } public InnerSequenceCompactionSelector getInnerSequenceCompactionSelector() { @@ -3157,14 +3156,6 @@ public void setInternalPort(int internalPort) { this.internalPort = internalPort; } - public int getAINodePort() { - return aiNodePort; - } - - public void setAINodePort(int aiNodePort) { - this.aiNodePort = aiNodePort; - } - public int getDataRegionConsensusPort() { return dataRegionConsensusPort; } @@ -3466,6 +3457,14 @@ public void setDataNodeSchemaCacheEvictionPolicy(String dataNodeSchemaCacheEvict this.dataNodeSchemaCacheEvictionPolicy = dataNodeSchemaCacheEvictionPolicy; } + public int getDataNodeTableCacheSemaphorePermitNum() { + return dataNodeTableCacheSemaphorePermitNum; + } + + public void setDataNodeTableCacheSemaphorePermitNum(int dataNodeTableCacheSemaphorePermitNum) { + this.dataNodeTableCacheSemaphorePermitNum = dataNodeTableCacheSemaphorePermitNum; + } + public String getReadConsistencyLevel() { return readConsistencyLevel; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 99ae453ce891..cb2067d9bea3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -102,6 +102,10 @@ public class IoTDBDescriptor { private static final double MIN_DIR_USE_PROPORTION = 0.5; + private static final String[] DEFAULT_WAL_THRESHOLD_NAME = { + "iot_consensus_throttle_threshold_in_byte", "wal_throttle_threshold_in_byte" + }; + static { URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); URL configNodeUrl = getPropsUrl(CommonConfig.OLD_CONFIG_NODE_CONFIG_NAME); @@ -287,19 +291,6 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO .getProperty(IoTDBConstant.DN_RPC_PORT, Integer.toString(conf.getRpcPort())) .trim())); - conf.setEnableAINodeService( - Boolean.parseBoolean( - properties - .getProperty( - "enable_ainode_rpc_service", Boolean.toString(conf.isEnableAINodeService())) - .trim())); - - conf.setAINodePort( - Integer.parseInt( - properties - .getProperty("ainode_rpc_port", Integer.toString(conf.getAINodePort())) - .trim())); - conf.setBufferedArraysMemoryProportion( Double.parseDouble( properties @@ -444,6 +435,12 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO "compaction_schedule_interval_in_ms", Long.toString(conf.getCompactionScheduleIntervalInMs())))); + conf.setEnableAutoRepairCompaction( + Boolean.parseBoolean( + properties.getProperty( + "enable_auto_repair_compaction", + Boolean.toString(conf.isEnableAutoRepairCompaction())))); + conf.setEnableCrossSpaceCompaction( Boolean.parseBoolean( properties.getProperty( @@ -1073,6 +1070,12 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO properties.getProperty( "datanode_schema_cache_eviction_policy", conf.getDataNodeSchemaCacheEvictionPolicy())); + conf.setDataNodeTableCacheSemaphorePermitNum( + Integer.parseInt( + properties.getProperty( + "datanode_table_cache_semaphore_permit_num", + String.valueOf(conf.getDataNodeTableCacheSemaphorePermitNum())))); + loadIoTConsensusProps(properties); loadIoTConsensusV2Props(properties); } @@ -1226,6 +1229,12 @@ private void loadCompactionHotModifiedProps(Properties properties) .setCompactionReadThroughputRate(conf.getCompactionReadThroughputMbPerSec()); CompactionTaskManager.getInstance() .setWriteMergeRate(conf.getCompactionWriteThroughputMbPerSec()); + + conf.setEnableAutoRepairCompaction( + Boolean.parseBoolean( + properties.getProperty( + "enable_auto_repair_compaction", + Boolean.toString(conf.isEnableAutoRepairCompaction())))); } private boolean loadCompactionTaskHotModifiedProps(Properties properties) throws IOException { @@ -1564,12 +1573,7 @@ private void loadWALHotModifiedProps(Properties properties) throws IOException { conf.setDeleteWalFilesPeriodInMs(deleteWalFilesPeriod); } - long throttleDownThresholdInByte = - Long.parseLong( - properties.getProperty( - "iot_consensus_throttle_threshold_in_byte", - ConfigurationFileUtils.getConfigurationDefaultValue( - "iot_consensus_throttle_threshold_in_byte"))); + long throttleDownThresholdInByte = Long.parseLong(getWalThrottleThreshold(properties)); if (throttleDownThresholdInByte > 0) { conf.setThrottleThreshold(throttleDownThresholdInByte); } @@ -1585,6 +1589,20 @@ private void loadWALHotModifiedProps(Properties properties) throws IOException { } } + private String getWalThrottleThreshold(Properties prop) throws IOException { + String old_throttleThreshold = prop.getProperty(DEFAULT_WAL_THRESHOLD_NAME[0], null); + if (old_throttleThreshold != null) { + LOGGER.warn( + "The throttle threshold params: {} is deprecated, please use {}", + DEFAULT_WAL_THRESHOLD_NAME[0], + DEFAULT_WAL_THRESHOLD_NAME[1]); + return old_throttleThreshold; + } + return prop.getProperty( + DEFAULT_WAL_THRESHOLD_NAME[1], + ConfigurationFileUtils.getConfigurationDefaultValue(DEFAULT_WAL_THRESHOLD_NAME[1])); + } + public long getThrottleThresholdWithDirs() { ArrayList dataDiskDirs = new ArrayList<>(Arrays.asList(conf.getDataDirs())); ArrayList walDiskDirs = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 29df33fbcc23..ebfa62888926 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -315,6 +315,18 @@ private void executeInternalCreateAlignedTimeSeries( encodingList.remove(index); compressionTypeList.remove(index); + // If with merge is set, the lists are deep copied and need to be altered here. + // We still remove the element from the original list to help cascading pipe transfer + // schema. + // If this exception is thrown, the measurements, data types, etc. must be unchanged. + // Thus, the index for the copied lists are identical to that in the original lists. + if (withMerge) { + createAlignedTimeSeriesPlan.getMeasurements().remove(index); + createAlignedTimeSeriesPlan.getDataTypes().remove(index); + createAlignedTimeSeriesPlan.getEncodings().remove(index); + createAlignedTimeSeriesPlan.getCompressors().remove(index); + } + if (measurementList.isEmpty()) { shouldRetry = false; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtask.java index 3f5529b02a16..6171e50b95c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/agent/task/subtask/connector/PipeConnectorSubtask.java @@ -254,7 +254,7 @@ public void discardEventsOfPipe(final String pipeNameToDrop, int regionId) { // "nonnull" detection. if (lastExceptionEvent instanceof EnrichedEvent && pipeNameToDrop.equals(((EnrichedEvent) lastExceptionEvent).getPipeName()) - && regionId == ((EnrichedEvent) lastEvent).getRegionId()) { + && regionId == ((EnrichedEvent) lastExceptionEvent).getRegionId()) { clearReferenceCountAndReleaseLastExceptionEvent(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeSyncClientManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeSyncClientManager.java index 5a341dc75745..5e4e0fbfcb8e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeSyncClientManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/client/IoTDBDataNodeSyncClientManager.java @@ -43,14 +43,14 @@ public class IoTDBDataNodeSyncClientManager extends IoTDBSyncClientManager LoggerFactory.getLogger(IoTDBDataNodeSyncClientManager.class); public IoTDBDataNodeSyncClientManager( - List endPoints, - boolean useSSL, - String trustStorePath, - String trustStorePwd, - boolean useLeaderCache, - String loadBalanceStrategy, - boolean shouldReceiverConvertOnTypeMismatch, - String loadTsFileStrategy) { + final List endPoints, + final boolean useSSL, + final String trustStorePath, + final String trustStorePwd, + final boolean useLeaderCache, + final String loadBalanceStrategy, + final boolean shouldReceiverConvertOnTypeMismatch, + final String loadTsFileStrategy) { super( endPoints, useSSL, @@ -69,7 +69,7 @@ protected PipeTransferDataNodeHandshakeV1Req buildHandshakeV1Req() throws IOExce } @Override - protected PipeTransferHandshakeV2Req buildHandshakeV2Req(Map params) + protected PipeTransferHandshakeV2Req buildHandshakeV2Req(final Map params) throws IOException { return PipeTransferDataNodeHandshakeV2Req.toTPipeTransferReq(params); } @@ -79,7 +79,7 @@ protected String getClusterId() { return IoTDBDescriptor.getInstance().getConfig().getClusterId(); } - public Pair getClient(String deviceId) { + public Pair getClient(final String deviceId) { final TEndPoint endPoint = LEADER_CACHE_MANAGER.getLeaderEndPoint(deviceId); return useLeaderCache && endPoint != null @@ -89,7 +89,7 @@ public Pair getClient(String deviceId) { : getClient(); } - public Pair getClient(TEndPoint endPoint) { + public Pair getClient(final TEndPoint endPoint) { return useLeaderCache && endPoint != null && endPoint2ClientAndStatus.containsKey(endPoint) @@ -98,7 +98,7 @@ public Pair getClient(TEndPoint endPoint) { : getClient(); } - public void updateLeaderCache(String deviceId, TEndPoint endPoint) { + public void updateLeaderCache(final String deviceId, final TEndPoint endPoint) { if (!useLeaderCache || deviceId == null || endPoint == null) { return; } @@ -111,7 +111,7 @@ public void updateLeaderCache(String deviceId, TEndPoint endPoint) { } LEADER_CACHE_MANAGER.updateLeaderEndPoint(deviceId, endPoint); - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn( "Failed to update leader cache for device {} with endpoint {}:{}.", deviceId, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataNodeSyncConnector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataNodeSyncConnector.java index 294e69a769ab..e6f1ac979577 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataNodeSyncConnector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/connector/protocol/thrift/sync/IoTDBDataNodeSyncConnector.java @@ -43,6 +43,7 @@ import java.net.UnknownHostException; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -93,7 +94,7 @@ protected IoTDBSyncClientManager constructClient( new IoTDBDataNodeSyncClientManager( nodeUrls, useSSL, - trustStorePath, + Objects.nonNull(trustStorePath) ? IoTDBConfig.addDataHomeDir(trustStorePath) : null, trustStorePwd, useLeaderCache, loadBalanceStrategy, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java index e25b4f4ad735..8c95d3489c9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tsfile/container/scan/TsFileInsertionScanDataContainer.java @@ -64,6 +64,8 @@ public class TsFileInsertionScanDataContainer extends TsFileInsertionDataContainer { + private static final LocalDate EMPTY_DATE = LocalDate.of(1000, 1, 1); + private final long startTime; private final long endTime; private final Filter filter; @@ -266,6 +268,13 @@ private void putValueToColumns(final BatchData data, final Tablet tablet, final final TsPrimitiveType primitiveType = data.getVector()[i]; if (Objects.isNull(primitiveType)) { tablet.bitMaps[i].mark(rowIndex); + final TSDataType type = tablet.getSchemas().get(i).getType(); + if (type == TSDataType.TEXT || type == TSDataType.BLOB || type == TSDataType.STRING) { + ((Binary[]) columns[i])[rowIndex] = Binary.EMPTY_VALUE; + } + if (type == TSDataType.DATE) { + ((LocalDate[]) columns[i])[rowIndex] = EMPTY_DATE; + } continue; } switch (tablet.getSchemas().get(i).getType()) { @@ -464,11 +473,13 @@ private void moveToNextChunkReader() throws IOException, IllegalStateException { private boolean recordAlignedChunk(final List valueChunkList, final byte marker) throws IOException { if (!valueChunkList.isEmpty()) { + final Chunk timeChunk = timeChunkList.get(lastIndex); + timeChunk.getData().rewind(); currentIsMultiPage = isMultiPageList.get(lastIndex); chunkReader = currentIsMultiPage - ? new AlignedChunkReader(timeChunkList.get(lastIndex), valueChunkList, filter) - : new AlignedSinglePageWholeChunkReader(timeChunkList.get(lastIndex), valueChunkList); + ? new AlignedChunkReader(timeChunk, valueChunkList, filter) + : new AlignedSinglePageWholeChunkReader(timeChunk, valueChunkList); currentIsAligned = true; lastMarker = marker; return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java index da61c57e58c2..39eb3b62a788 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/twostage/plugin/TwoStageCountProcessor.java @@ -240,7 +240,7 @@ private void collectGlobalCountIfNecessary(EventCollector eventCollector) throws final Tablet tablet = new Tablet( - outputSeries.getDeviceString(), + outputSeries.getIDeviceID().toString(), Collections.singletonList( new MeasurementSchema(outputSeries.getMeasurement(), TSDataType.INT64)), 1); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java index db38102ca9d5..09e2f265ec55 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/transform/converter/ValueConverter.java @@ -527,11 +527,11 @@ public static boolean convertTextToBoolean(final Binary value) { } public static int convertTextToInt32(final Binary value) { - return parseInteger(value.toString()); + return (int) parseDouble(value.toString()); } public static long convertTextToInt64(final Binary value) { - return parseLong(value.toString()); + return (long) parseDouble(value.toString()); } public static float convertTextToFloat(final Binary value) { @@ -653,11 +653,11 @@ public static boolean convertBlobToBoolean(final Binary value) { } public static int convertBlobToInt32(final Binary value) { - return parseInteger(value.toString()); + return (int) parseDouble(value.toString()); } public static long convertBlobToInt64(final Binary value) { - return parseLong(value.toString()); + return (long) parseDouble(value.toString()); } public static float convertBlobToFloat(final Binary value) { @@ -691,11 +691,11 @@ public static boolean convertStringToBoolean(final Binary value) { } public static int convertStringToInt32(final Binary value) { - return parseInteger(value.toString()); + return (int) parseDouble(value.toString()); } public static long convertStringToInt64(final Binary value) { - return parseLong(value.toString()); + return (long) parseDouble(value.toString()); } public static float convertStringToFloat(final Binary value) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 3df6f4829bc5..f3c6a07307d1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -96,6 +96,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; +import org.apache.iotdb.confignode.rpc.thrift.TFetchTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllSubscriptionInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TGetAllTemplatesResp; @@ -180,6 +181,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.function.Predicate; public class ConfigNodeClient implements IConfigNodeRPCService.Iface, ThriftClient, AutoCloseable { @@ -1278,6 +1280,13 @@ public TShowTableResp showTables(final String database) throws TException { () -> client.showTables(database), resp -> !updateConfigNodeLeader(resp.status)); } + @Override + public TFetchTableResp fetchTables(final Map> fetchTableMap) + throws TException { + return executeRemoteCallWithRetry( + () -> client.fetchTables(fetchTableMap), resp -> !updateConfigNodeLeader(resp.status)); + } + public static class Factory extends ThriftClientFactory { public Factory( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/handler/AINodeRPCServiceThriftHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/handler/AINodeRPCServiceThriftHandler.java deleted file mode 100644 index c5969f8678a3..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/handler/AINodeRPCServiceThriftHandler.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.iotdb.db.protocol.thrift.handler; - -import org.apache.iotdb.db.protocol.thrift.impl.IAINodeRPCServiceWithHandler; - -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.server.ServerContext; -import org.apache.thrift.server.TServerEventHandler; -import org.apache.thrift.transport.TTransport; - -import java.util.concurrent.atomic.AtomicLong; - -public class AINodeRPCServiceThriftHandler implements TServerEventHandler { - - private final AtomicLong thriftConnectionNumber = new AtomicLong(0); - private final IAINodeRPCServiceWithHandler eventHandler; - - public AINodeRPCServiceThriftHandler(IAINodeRPCServiceWithHandler eventHandler) { - this.eventHandler = eventHandler; - } - - @Override - public ServerContext createContext(TProtocol in, TProtocol out) { - thriftConnectionNumber.incrementAndGet(); - return null; - } - - @Override - public void deleteContext(ServerContext arg0, TProtocol in, TProtocol out) { - thriftConnectionNumber.decrementAndGet(); - eventHandler.handleExit(); - } - - @Override - public void preServe() { - // do nothing - } - - @Override - public void processContext( - ServerContext serverContext, TTransport tTransport, TTransport tTransport1) { - // do nothing - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/AINodeRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/AINodeRPCServiceImpl.java deleted file mode 100644 index 68e492cdf462..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/AINodeRPCServiceImpl.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.protocol.thrift.impl; - -import org.apache.iotdb.common.rpc.thrift.TSStatus; -import org.apache.iotdb.commons.conf.IoTDBConstant.ClientVersion; -import org.apache.iotdb.db.protocol.session.IClientSession; -import org.apache.iotdb.db.protocol.session.InternalClientSession; -import org.apache.iotdb.db.protocol.session.SessionManager; -import org.apache.iotdb.db.protocol.thrift.OperationType; -import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; -import org.apache.iotdb.db.queryengine.plan.Coordinator; -import org.apache.iotdb.db.queryengine.plan.analyze.ClusterPartitionFetcher; -import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher; -import org.apache.iotdb.db.queryengine.plan.analyze.schema.ClusterSchemaFetcher; -import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher; -import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; -import org.apache.iotdb.db.queryengine.plan.execution.IQueryExecution; -import org.apache.iotdb.db.queryengine.plan.parser.StatementGenerator; -import org.apache.iotdb.db.queryengine.plan.statement.Statement; -import org.apache.iotdb.db.utils.ErrorHandlingUtils; -import org.apache.iotdb.db.utils.QueryDataSetUtils; -import org.apache.iotdb.db.utils.SetThreadName; -import org.apache.iotdb.mpp.rpc.thrift.TFetchMoreDataReq; -import org.apache.iotdb.mpp.rpc.thrift.TFetchMoreDataResp; -import org.apache.iotdb.mpp.rpc.thrift.TFetchTimeseriesReq; -import org.apache.iotdb.mpp.rpc.thrift.TFetchTimeseriesResp; -import org.apache.iotdb.rpc.RpcUtils; -import org.apache.iotdb.rpc.TSStatusCode; - -import org.apache.thrift.TException; -import org.apache.tsfile.utils.Pair; - -import java.nio.ByteBuffer; -import java.time.ZoneId; -import java.util.List; - -public class AINodeRPCServiceImpl implements IAINodeRPCServiceWithHandler { - - public static final String AI_METRICS_PATH_PREFIX = "root.__system.AI.exp"; - - private static final SessionManager SESSION_MANAGER = SessionManager.getInstance(); - - private static final Coordinator COORDINATOR = Coordinator.getInstance(); - - private final IPartitionFetcher partitionFetcher; - - private final ISchemaFetcher schemaFetcher; - - private final IClientSession session; - - public AINodeRPCServiceImpl() { - super(); - partitionFetcher = ClusterPartitionFetcher.getInstance(); - schemaFetcher = ClusterSchemaFetcher.getInstance(); - session = new InternalClientSession("AINodeService"); - SESSION_MANAGER.registerSession(session); - SESSION_MANAGER.supplySession(session, "AINode", ZoneId.systemDefault(), ClientVersion.V_1_0); - } - - @Override - public TFetchTimeseriesResp fetchTimeseries(TFetchTimeseriesReq req) throws TException { - boolean finished = false; - TFetchTimeseriesResp resp = new TFetchTimeseriesResp(); - Throwable t = null; - try { - - Statement s = StatementGenerator.createStatement(req, session.getZoneId()); - - if (s == null) { - resp.setStatus( - RpcUtils.getStatus( - TSStatusCode.SQL_PARSE_ERROR, "This operation type is not supported")); - return resp; - } - - long queryId = - SESSION_MANAGER.requestQueryId(session, SESSION_MANAGER.requestStatementId(session)); - ExecutionResult result = - COORDINATOR.executeForTreeModel( - s, - queryId, - SESSION_MANAGER.getSessionInfo(session), - "", - partitionFetcher, - schemaFetcher, - req.getTimeout()); - - if (result.status.code != TSStatusCode.SUCCESS_STATUS.getStatusCode() - && result.status.code != TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) { - resp.setStatus(result.status); - return resp; - } - - IQueryExecution queryExecution = COORDINATOR.getQueryExecution(queryId); - - try (SetThreadName threadName = new SetThreadName(result.queryId.getId())) { - - DatasetHeader header = queryExecution.getDatasetHeader(); - resp.setStatus(result.status); - resp.setColumnNameList(header.getRespColumns()); - resp.setColumnTypeList(header.getRespDataTypeList()); - resp.setColumnNameIndexMap(header.getColumnNameIndexMap()); - resp.setQueryId(queryId); - - Pair, Boolean> pair = - QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, req.fetchSize); - resp.setTsDataset(pair.left); - finished = pair.right; - resp.setHasMoreData(!finished); - return resp; - } - } catch (Exception e) { - finished = true; - t = e; - resp.setStatus(ErrorHandlingUtils.onQueryException(e, OperationType.EXECUTE_STATEMENT)); - return resp; - } catch (Error error) { - t = error; - throw error; - } finally { - if (finished) { - COORDINATOR.cleanupQueryExecution(resp.queryId, req, t); - } - } - } - - @Override - public TFetchMoreDataResp fetchMoreData(TFetchMoreDataReq req) throws TException { - TFetchMoreDataResp resp = new TFetchMoreDataResp(); - boolean finished = false; - Throwable t = null; - try { - IQueryExecution queryExecution = COORDINATOR.getQueryExecution(req.queryId); - resp.setStatus(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())); - - if (queryExecution == null) { - resp.setHasMoreData(false); - return resp; - } - - try (SetThreadName queryName = new SetThreadName(queryExecution.getQueryId())) { - Pair, Boolean> pair = - QueryDataSetUtils.convertQueryResultByFetchSize(queryExecution, req.fetchSize); - List result = pair.left; - finished = pair.right; - resp.setTsDataset(result); - resp.setHasMoreData(!finished); - return resp; - } - } catch (Exception e) { - finished = true; - t = e; - resp.setStatus(ErrorHandlingUtils.onQueryException(e, OperationType.FETCH_RESULTS)); - return resp; - } catch (Error error) { - t = error; - throw error; - } finally { - if (finished) { - COORDINATOR.cleanupQueryExecution(req.queryId, req, t); - } - } - } - - @Override - public void handleExit() { - SESSION_MANAGER.closeSession(session, COORDINATOR::cleanupQueryExecution); - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 163137babf68..c28ff6f9b791 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -522,6 +522,7 @@ public TSStatus invalidateSchemaCache(TInvalidateCacheReq req) { String database = req.getFullPath().substring(5); DataNodeTableCache.getInstance().invalid(database); TableDeviceSchemaFetcher.getInstance().getTableDeviceCache().invalidate(database); + LOGGER.info("Schema cache of {} has been invalidated", req.getFullPath()); return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } finally { DataNodeSchemaCache.getInstance().releaseWriteLock(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java index 4ef11066229d..7bc66103e661 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java @@ -33,11 +33,11 @@ import org.apache.tsfile.read.filter.basic.Filter; import java.time.ZoneId; -import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Optional; +import java.util.Set; /** * This class is used to record the context of a query including QueryId, query statement, session @@ -72,7 +72,7 @@ public class MPPQueryContext { private Filter globalTimeFilter; - private Map acquiredLockNumMap = new HashMap<>(); + private final Set acquiredLocks = new HashSet<>(); private boolean isExplainAnalyze = false; @@ -203,16 +203,12 @@ public String getSql() { return sql; } - public Map getAcquiredLockNumMap() { - return acquiredLockNumMap; + public Set getAcquiredLocks() { + return acquiredLocks; } - public void addAcquiredLockNum(SchemaLockType lockType) { - if (acquiredLockNumMap.containsKey(lockType)) { - acquiredLockNumMap.put(lockType, acquiredLockNumMap.get(lockType) + 1); - } else { - acquiredLockNumMap.put(lockType, 1); - } + public boolean addAcquiredLock(final SchemaLockType lockType) { + return acquiredLocks.add(lockType); } // used for tree model diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionReadExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionReadExecutor.java index 68ee5352cf06..b045babc6d63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionReadExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/executor/RegionReadExecutor.java @@ -118,7 +118,8 @@ public RegionExecutionResult execute( if (t instanceof ReadException || t instanceof ReadIndexException || t instanceof NotLeaderException - || t instanceof ServerNotReadyException) { + || t instanceof ServerNotReadyException + || t instanceof InterruptedException) { resp.setReadNeedRetry(true); resp.setStatus(new TSStatus(TSStatusCode.RATIS_READ_UNAVAILABLE.getStatusCode())); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index a2a3021a51b5..94917b12cd63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -406,9 +406,8 @@ public void setTimeFilterForTableModel(Filter timeFilter) { if (globalTimeFilter == null) { globalTimeFilter = timeFilter; } else { - // In join case, there may exist more than one table and time filter, join criteria only - // support and condition, so use and to connect these two filters - globalTimeFilter = FilterFactory.and(globalTimeFilter, timeFilter); + // In join case, there may exist more than one table and time filter + globalTimeFilter = FilterFactory.or(globalTimeFilter, timeFilter); // throw new IllegalStateException( // "globalTimeFilter in FragmentInstanceContext should only be set once in Table Model!"); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AggregationMergeSortOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AggregationMergeSortOperator.java index cd1d53320283..f67975d84fbd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AggregationMergeSortOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/AggregationMergeSortOperator.java @@ -147,7 +147,7 @@ public TsBlock next() throws Exception { outputResultToTsBlock(); } - return tsBlockBuilder.build(); + return tsBlockBuilder.getPositionCount() > 0 ? tsBlockBuilder.build() : null; } private void outputResultToTsBlock() { @@ -160,6 +160,7 @@ private void outputResultToTsBlock() { } tsBlockBuilder.declarePosition(); accumulators.forEach(Accumulator::reset); + lastDevice = null; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java new file mode 100644 index 000000000000..e5b669f8300d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableFullOuterJoinOperator.java @@ -0,0 +1,521 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source.relational; + +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.queryengine.execution.operator.AbstractOperator; +import org.apache.iotdb.db.queryengine.execution.operator.Operator; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.TimeComparator; +import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static com.google.common.util.concurrent.Futures.successfulAsList; +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableInnerJoinOperator.buildResultTsBlock; + +public class TableFullOuterJoinOperator extends AbstractOperator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(TableFullOuterJoinOperator.class); + + private final Operator leftChild; + private TsBlock leftBlock; + private final int leftTimeColumnPosition; + private int leftIndex; // start index of leftTsBlock + private final int[] leftOutputSymbolIdx; + private boolean leftFinished; + + private final Operator rightChild; + private final List rightBlockList = new ArrayList<>(); + private final int rightTimeColumnPosition; + private int rightBlockListIdx; + private int rightIndex; // start index of rightTsBlock + private final int[] rightOutputSymbolIdx; + private TsBlock cachedNextRightBlock; + private boolean hasCachedNextRightBlock; + private boolean rightFinished; + private long lastMatchedRightTime = Long.MIN_VALUE; + + private final TimeComparator comparator; + private final TsBlockBuilder resultBuilder; + + protected MemoryReservationManager memoryReservationManager; + + public TableFullOuterJoinOperator( + OperatorContext operatorContext, + Operator leftChild, + int leftTimeColumnPosition, + int[] leftOutputSymbolIdx, + Operator rightChild, + int rightTimeColumnPosition, + int[] rightOutputSymbolIdx, + TimeComparator timeComparator, + List dataTypes) { + this.operatorContext = operatorContext; + this.leftChild = leftChild; + this.leftTimeColumnPosition = leftTimeColumnPosition; + this.leftOutputSymbolIdx = leftOutputSymbolIdx; + this.rightChild = rightChild; + this.rightOutputSymbolIdx = rightOutputSymbolIdx; + this.rightTimeColumnPosition = rightTimeColumnPosition; + + this.comparator = timeComparator; + this.resultBuilder = new TsBlockBuilder(dataTypes); + + this.memoryReservationManager = + operatorContext + .getDriverContext() + .getFragmentInstanceContext() + .getMemoryReservationContext(); + } + + @Override + public ListenableFuture isBlocked() { + ListenableFuture leftBlocked = leftChild.isBlocked(); + ListenableFuture rightBlocked = rightChild.isBlocked(); + if (leftBlocked.isDone()) { + return rightBlocked; + } else if (rightBlocked.isDone()) { + return leftBlocked; + } else { + return successfulAsList(leftBlocked, rightBlocked); + } + } + + @Override + public boolean isFinished() throws Exception { + if (retainedTsBlock != null) { + return false; + } + + return !leftBlockNotEmpty() + && leftChild.isFinished() + && !rightBlockNotEmpty() + && rightChild.isFinished(); + } + + @Override + public boolean hasNext() throws Exception { + if (retainedTsBlock != null) { + return true; + } + + return (leftBlockNotEmpty() || leftChild.hasNextWithTimer()) + || (rightBlockNotEmpty() || rightChild.hasNextWithTimer()); + } + + @Override + public TsBlock next() throws Exception { + if (retainedTsBlock != null) { + return getResultFromRetainedTsBlock(); + } + resultBuilder.reset(); + + long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS); + long start = System.nanoTime(); + // prepare leftBlock and rightBlockList with cachedNextRightBlock + if (!prepareInput(start, maxRuntime)) { + return null; + } + + if (leftFinished || rightFinished) { + if (leftFinished) { + appendRightWithEmptyLeft(); + for (int i = 1; i < rightBlockList.size(); i++) { + memoryReservationManager.releaseMemoryCumulatively( + rightBlockList.get(i).getRetainedSizeInBytes()); + } + rightBlockList.clear(); + rightBlockListIdx = 0; + rightIndex = 0; + resultTsBlock = buildResultTsBlock(resultBuilder); + return checkTsBlockSizeAndGetResult(); + } else { + appendLeftWithEmptyRight(); + leftBlock = null; + leftIndex = 0; + resultTsBlock = buildResultTsBlock(resultBuilder); + return checkTsBlockSizeAndGetResult(); + } + } + + // all the rightTsBlock is less than leftTsBlock, append right with empty left + if (comparator.lessThan(getRightEndTime(), getCurrentLeftTime())) { + appendRightWithEmptyLeft(); + for (int i = 1; i < rightBlockList.size(); i++) { + memoryReservationManager.releaseMemoryCumulatively( + rightBlockList.get(i).getRetainedSizeInBytes()); + } + rightBlockList.clear(); + rightBlockListIdx = 0; + rightIndex = 0; + return null; + } + + // all the leftTsBlock is less than rightTsBlock, append left with empty right + else if (comparator.lessThan(getLeftEndTime(), getCurrentRightTime())) { + appendLeftWithEmptyRight(); + leftBlock = null; + leftIndex = 0; + return null; + } + + long leftProbeTime = getCurrentLeftTime(); + while (!resultBuilder.isFull()) { + + // all right block time is not matched + if (!comparator.canContinueInclusive(leftProbeTime, getRightEndTime())) { + appendRightWithEmptyLeft(); + for (int i = 1; i < rightBlockList.size(); i++) { + memoryReservationManager.releaseMemoryCumulatively( + rightBlockList.get(i).getRetainedSizeInBytes()); + } + rightBlockList.clear(); + rightBlockListIdx = 0; + rightIndex = 0; + break; + } + + appendResult(leftProbeTime); + + leftIndex++; + + if (leftIndex >= leftBlock.getPositionCount()) { + leftBlock = null; + leftIndex = 0; + break; + } + + leftProbeTime = getCurrentLeftTime(); + } + + if (resultBuilder.isEmpty()) { + return null; + } + + resultTsBlock = buildResultTsBlock(resultBuilder); + return checkTsBlockSizeAndGetResult(); + } + + private boolean prepareInput(long start, long maxRuntime) throws Exception { + + if (!leftFinished && (leftBlock == null || leftBlock.getPositionCount() == leftIndex)) { + if (leftChild.hasNextWithTimer()) { + leftBlock = leftChild.nextWithTimer(); + leftIndex = 0; + } else { + leftFinished = true; + } + } + + if (!rightFinished) { + if (rightBlockList.isEmpty()) { + if (hasCachedNextRightBlock && cachedNextRightBlock != null) { + rightBlockList.add(cachedNextRightBlock); + hasCachedNextRightBlock = false; + cachedNextRightBlock = null; + tryCachedNextRightTsBlock(); + } else if (rightChild.hasNextWithTimer()) { + TsBlock block = rightChild.nextWithTimer(); + if (block != null) { + rightBlockList.add(block); + tryCachedNextRightTsBlock(); + } + } else { + rightFinished = true; + hasCachedNextRightBlock = true; + cachedNextRightBlock = null; + } + } else { + if (!hasCachedNextRightBlock) { + tryCachedNextRightTsBlock(); + } + } + } + + return (leftBlockNotEmpty() && rightBlockNotEmpty() && hasCachedNextRightBlock) + || (leftBlockNotEmpty() && rightFinished) + || (leftFinished && rightBlockNotEmpty() && hasCachedNextRightBlock); + } + + private void tryCachedNextRightTsBlock() throws Exception { + if (rightChild.hasNextWithTimer()) { + TsBlock block = rightChild.nextWithTimer(); + if (block != null) { + if (block.getColumn(rightTimeColumnPosition).getLong(0) == getRightEndTime()) { + memoryReservationManager.reserveMemoryCumulatively(block.getRetainedSizeInBytes()); + rightBlockList.add(block); + } else { + hasCachedNextRightBlock = true; + cachedNextRightBlock = block; + } + } + } else { + hasCachedNextRightBlock = true; + cachedNextRightBlock = null; + } + } + + private long getCurrentLeftTime() { + return leftBlock.getColumn(leftTimeColumnPosition).getLong(leftIndex); + } + + private long getLeftEndTime() { + return leftBlock.getColumn(leftTimeColumnPosition).getLong(leftBlock.getPositionCount() - 1); + } + + private long getCurrentRightTime() { + return rightBlockList + .get(rightBlockListIdx) + .getColumn(rightTimeColumnPosition) + .getLong(rightIndex); + } + + private long getRightTime(int idx1, int idx2) { + return rightBlockList.get(idx1).getColumn(rightTimeColumnPosition).getLong(idx2); + } + + private long getRightEndTime() { + TsBlock lastRightTsBlock = rightBlockList.get(rightBlockList.size() - 1); + return lastRightTsBlock + .getColumn(rightTimeColumnPosition) + .getLong(lastRightTsBlock.getPositionCount() - 1); + } + + private void appendResult(long leftTime) { + + while (comparator.lessThan(getCurrentRightTime(), leftTime)) { + if (getCurrentRightTime() > lastMatchedRightTime) { + appendOneRightRowWithEmptyLeft(); + } + + rightIndex++; + + if (rightIndex >= rightBlockList.get(rightBlockListIdx).getPositionCount()) { + rightBlockListIdx++; + rightIndex = 0; + } + + if (rightBlockListIdx >= rightBlockList.size()) { + rightBlockListIdx = 0; + rightIndex = 0; + return; + } + } + + int tmpBlockIdx = rightBlockListIdx, tmpIdx = rightIndex; + while (leftTime == getRightTime(tmpBlockIdx, tmpIdx)) { + // lastMatchedRightBlockListIdx = rightBlockListIdx; + // lastMatchedRightIdx = rightIndex; + lastMatchedRightTime = leftTime; + appendValueToResult(tmpBlockIdx, tmpIdx); + + resultBuilder.declarePosition(); + + tmpIdx++; + if (tmpIdx >= rightBlockList.get(tmpBlockIdx).getPositionCount()) { + tmpIdx = 0; + tmpBlockIdx++; + } + + if (tmpBlockIdx >= rightBlockList.size()) { + break; + } + } + } + + private void appendLeftWithEmptyRight() { + while (leftIndex < leftBlock.getPositionCount()) { + for (int i = 0; i < leftOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(i); + if (leftBlock.getColumn(leftOutputSymbolIdx[i]).isNull(leftIndex)) { + columnBuilder.appendNull(); + } else { + columnBuilder.write(leftBlock.getColumn(leftOutputSymbolIdx[i]), leftIndex); + } + } + + for (int i = 0; i < rightOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = + resultBuilder.getColumnBuilder(leftOutputSymbolIdx.length + i); + columnBuilder.appendNull(); + } + + resultBuilder.declarePosition(); + leftIndex++; + } + } + + private void appendRightWithEmptyLeft() { + while (rightBlockListIdx < rightBlockList.size()) { + + if (getCurrentRightTime() > lastMatchedRightTime) { + for (int i = 0; i < leftOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(i); + columnBuilder.appendNull(); + } + + for (int i = 0; i < rightOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = + resultBuilder.getColumnBuilder(leftOutputSymbolIdx.length + i); + + if (rightBlockList + .get(rightBlockListIdx) + .getColumn(rightOutputSymbolIdx[i]) + .isNull(rightIndex)) { + columnBuilder.appendNull(); + } else { + columnBuilder.write( + rightBlockList.get(rightBlockListIdx).getColumn(rightOutputSymbolIdx[i]), + rightIndex); + } + } + + resultBuilder.declarePosition(); + } + + rightIndex++; + if (rightIndex >= rightBlockList.get(rightBlockListIdx).getPositionCount()) { + rightIndex = 0; + rightBlockListIdx++; + } + } + } + + private void appendOneRightRowWithEmptyLeft() { + for (int i = 0; i < leftOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(i); + columnBuilder.appendNull(); + } + + for (int i = 0; i < rightOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(leftOutputSymbolIdx.length + i); + + if (rightBlockList + .get(rightBlockListIdx) + .getColumn(rightOutputSymbolIdx[i]) + .isNull(rightIndex)) { + columnBuilder.appendNull(); + } else { + columnBuilder.write( + rightBlockList.get(rightBlockListIdx).getColumn(rightOutputSymbolIdx[i]), rightIndex); + } + } + + resultBuilder.declarePosition(); + } + + private boolean leftBlockNotEmpty() { + return leftBlock != null && leftIndex < leftBlock.getPositionCount(); + } + + private boolean rightBlockNotEmpty() { + return !rightBlockList.isEmpty() + && rightBlockListIdx < rightBlockList.size() + && rightIndex < rightBlockList.get(rightBlockListIdx).getPositionCount(); + } + + private void appendValueToResult(int tmpRightBlockListIdx, int tmpRightIndex) { + for (int i = 0; i < leftOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(i); + if (leftBlock.getColumn(leftOutputSymbolIdx[i]).isNull(leftIndex)) { + columnBuilder.appendNull(); + } else { + columnBuilder.write(leftBlock.getColumn(leftOutputSymbolIdx[i]), leftIndex); + } + } + + for (int i = 0; i < rightOutputSymbolIdx.length; i++) { + ColumnBuilder columnBuilder = resultBuilder.getColumnBuilder(leftOutputSymbolIdx.length + i); + + if (rightBlockList + .get(tmpRightBlockListIdx) + .getColumn(rightOutputSymbolIdx[i]) + .isNull(tmpRightIndex)) { + columnBuilder.appendNull(); + } else { + columnBuilder.write( + rightBlockList.get(tmpRightBlockListIdx).getColumn(rightOutputSymbolIdx[i]), + tmpRightIndex); + } + } + } + + @Override + public void close() throws Exception { + if (leftChild != null) { + leftChild.close(); + } + if (rightChild != null) { + rightChild.close(); + } + + if (!rightBlockList.isEmpty()) { + for (TsBlock block : rightBlockList) { + memoryReservationManager.reserveMemoryCumulatively(block.getRetainedSizeInBytes()); + } + } + } + + @Override + public long calculateMaxPeekMemory() { + return Math.max( + Math.max( + leftChild.calculateMaxPeekMemoryWithCounter(), + rightChild.calculateMaxPeekMemoryWithCounter()), + calculateRetainedSizeAfterCallingNext() + calculateMaxReturnSize()); + } + + @Override + public long calculateMaxReturnSize() { + return maxReturnSize * 2; + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + // leftTsBlock + leftChild.RetainedSizeAfterCallingNext + rightTsBlock + + // rightChild.RetainedSizeAfterCallingNext + return leftChild.calculateMaxReturnSize() + + leftChild.calculateRetainedSizeAfterCallingNext() + + rightChild.calculateMaxReturnSize() + + rightChild.calculateRetainedSizeAfterCallingNext() + + maxReturnSize; + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(leftChild) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(rightChild) + + RamUsageEstimator.sizeOf(leftOutputSymbolIdx) + + RamUsageEstimator.sizeOf(rightOutputSymbolIdx) + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) + + resultBuilder.getRetainedSizeInBytes(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InnerJoinOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableInnerJoinOperator.java similarity index 87% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InnerJoinOperator.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableInnerJoinOperator.java index c086666ece5b..73963c2359bc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InnerJoinOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/TableInnerJoinOperator.java @@ -20,16 +20,15 @@ package org.apache.iotdb.db.queryengine.execution.operator.source.relational; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.queryengine.execution.operator.AbstractOperator; import org.apache.iotdb.db.queryengine.execution.operator.Operator; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; -import org.apache.iotdb.db.queryengine.execution.operator.process.ProcessOperator; import org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.TimeComparator; import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; import com.google.common.util.concurrent.ListenableFuture; import org.apache.tsfile.block.column.Column; import org.apache.tsfile.block.column.ColumnBuilder; -import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; @@ -43,20 +42,19 @@ import static com.google.common.util.concurrent.Futures.successfulAsList; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator.TIME_COLUMN_TEMPLATE; -public class InnerJoinOperator implements ProcessOperator { +public class TableInnerJoinOperator extends AbstractOperator { private static final long INSTANCE_SIZE = - RamUsageEstimator.shallowSizeOfInstance(InnerJoinOperator.class); - - private final OperatorContext operatorContext; + RamUsageEstimator.shallowSizeOfInstance(TableInnerJoinOperator.class); private final Operator leftChild; private TsBlock leftBlock; private int leftIndex; // start index of leftTsBlock - private static final int TIME_COLUMN_POSITION = 0; + private final int leftTimeColumnPosition; private final int[] leftOutputSymbolIdx; private final Operator rightChild; private final List rightBlockList = new ArrayList<>(); + private final int rightTimeColumnPosition; private int rightBlockListIdx; private int rightIndex; // start index of rightTsBlock private final int[] rightOutputSymbolIdx; @@ -66,23 +64,24 @@ public class InnerJoinOperator implements ProcessOperator { private final TimeComparator comparator; private final TsBlockBuilder resultBuilder; - private final long maxReturnSize = - TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); - protected MemoryReservationManager memoryReservationManager; - public InnerJoinOperator( + public TableInnerJoinOperator( OperatorContext operatorContext, Operator leftChild, + int leftTimeColumnPosition, int[] leftOutputSymbolIdx, Operator rightChild, + int rightTimeColumnPosition, int[] rightOutputSymbolIdx, TimeComparator timeComparator, List dataTypes) { this.operatorContext = operatorContext; this.leftChild = leftChild; + this.leftTimeColumnPosition = leftTimeColumnPosition; this.leftOutputSymbolIdx = leftOutputSymbolIdx; this.rightChild = rightChild; + this.rightTimeColumnPosition = rightTimeColumnPosition; this.rightOutputSymbolIdx = rightOutputSymbolIdx; this.comparator = timeComparator; @@ -108,14 +107,35 @@ public ListenableFuture isBlocked() { } } + @Override + public boolean isFinished() throws Exception { + if (retainedTsBlock != null) { + return true; + } + + return !leftBlockNotEmpty() + && leftChild.isFinished() + && !rightBlockNotEmpty() + && rightChild.isFinished(); + } + @Override public boolean hasNext() throws Exception { + if (retainedTsBlock != null) { + return true; + } + return (leftBlockNotEmpty() || leftChild.hasNextWithTimer()) && (rightBlockNotEmpty() || rightChild.hasNextWithTimer()); } @Override public TsBlock next() throws Exception { + if (retainedTsBlock != null) { + return getResultFromRetainedTsBlock(); + } + resultBuilder.reset(); + long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS); long start = System.nanoTime(); // prepare leftBlock and rightBlockList with cachedNextRightBlock @@ -136,7 +156,7 @@ public TsBlock next() throws Exception { } // all the leftTsBlock is less than rightTsBlock, just skip left - else if (comparator.lessThan(getLeftEndTime(), getCurrentRightTime())) { + else if (comparator.lessThan(getLeftEndTime(), getRightTime(rightBlockListIdx, rightIndex))) { leftBlock = null; leftIndex = 0; return null; @@ -174,24 +194,8 @@ else if (comparator.lessThan(getLeftEndTime(), getCurrentRightTime())) { return null; } - Column[] valueColumns = new Column[resultBuilder.getValueColumnBuilders().length]; - for (int i = 0; i < valueColumns.length; ++i) { - valueColumns[i] = resultBuilder.getValueColumnBuilders()[i].build(); - if (valueColumns[i].getPositionCount() != resultBuilder.getPositionCount()) { - throw new IllegalStateException( - String.format( - "Declared positions (%s) does not match column %s's number of entries (%s)", - resultBuilder.getPositionCount(), i, valueColumns[i].getPositionCount())); - } - } - - TsBlock result = - TsBlock.wrapBlocksWithoutCopy( - this.resultBuilder.getPositionCount(), - new RunLengthEncodedColumn(TIME_COLUMN_TEMPLATE, this.resultBuilder.getPositionCount()), - valueColumns); - resultBuilder.reset(); - return result; + resultTsBlock = buildResultTsBlock(resultBuilder); + return checkTsBlockSizeAndGetResult(); } private boolean prepareInput(long start, long maxRuntime) throws Exception { @@ -230,7 +234,7 @@ private void tryCachedNextRightTsBlock() throws Exception { if (rightChild.hasNextWithTimer()) { TsBlock block = rightChild.nextWithTimer(); if (block != null) { - if (block.getColumn(TIME_COLUMN_POSITION).getLong(0) == getRightEndTime()) { + if (block.getColumn(rightTimeColumnPosition).getLong(0) == getRightEndTime()) { memoryReservationManager.reserveMemoryCumulatively(block.getRetainedSizeInBytes()); rightBlockList.add(block); } else { @@ -245,28 +249,25 @@ private void tryCachedNextRightTsBlock() throws Exception { } private long getCurrentLeftTime() { - return leftBlock.getColumn(TIME_COLUMN_POSITION).getLong(leftIndex); + return leftBlock.getColumn(leftTimeColumnPosition).getLong(leftIndex); } private long getLeftEndTime() { - return leftBlock.getColumn(TIME_COLUMN_POSITION).getLong(leftBlock.getPositionCount() - 1); + return leftBlock.getColumn(leftTimeColumnPosition).getLong(leftBlock.getPositionCount() - 1); } - private long getCurrentRightTime() { - return rightBlockList - .get(rightBlockListIdx) - .getColumn(TIME_COLUMN_POSITION) - .getLong(rightIndex); + private long getRightTime(int blockIdx, int rowIdx) { + return rightBlockList.get(blockIdx).getColumn(rightTimeColumnPosition).getLong(rowIdx); } - private long getRightTime(int idx1, int idx2) { - return rightBlockList.get(idx1).getColumn(TIME_COLUMN_POSITION).getLong(idx2); + private long getCurrentRightTime() { + return getRightTime(rightBlockListIdx, rightIndex); } private long getRightEndTime() { TsBlock lastRightTsBlock = rightBlockList.get(rightBlockList.size() - 1); return lastRightTsBlock - .getColumn(TIME_COLUMN_POSITION) + .getColumn(rightTimeColumnPosition) .getLong(lastRightTsBlock.getPositionCount() - 1); } @@ -341,12 +342,25 @@ private void appendValueToResult(int tmpRightBlockListIdx, int tmpRightIndex) { } } - @Override - public boolean isFinished() throws Exception { - return !leftBlockNotEmpty() - && leftChild.isFinished() - && !rightBlockNotEmpty() - && rightChild.isFinished(); + public static TsBlock buildResultTsBlock(TsBlockBuilder resultBuilder) { + Column[] valueColumns = new Column[resultBuilder.getValueColumnBuilders().length]; + for (int i = 0; i < valueColumns.length; ++i) { + valueColumns[i] = resultBuilder.getValueColumnBuilders()[i].build(); + if (valueColumns[i].getPositionCount() != resultBuilder.getPositionCount()) { + throw new IllegalStateException( + String.format( + "Declared positions (%s) does not match column %s's number of entries (%s)", + resultBuilder.getPositionCount(), i, valueColumns[i].getPositionCount())); + } + } + + TsBlock result = + TsBlock.wrapBlocksWithoutCopy( + resultBuilder.getPositionCount(), + new RunLengthEncodedColumn(TIME_COLUMN_TEMPLATE, resultBuilder.getPositionCount()), + valueColumns); + resultBuilder.reset(); + return result; } @Override @@ -365,11 +379,6 @@ public void close() throws Exception { } } - @Override - public OperatorContext getOperatorContext() { - return operatorContext; - } - @Override public long calculateMaxPeekMemory() { return Math.max( @@ -391,7 +400,8 @@ public long calculateRetainedSizeAfterCallingNext() { return leftChild.calculateMaxReturnSize() + leftChild.calculateRetainedSizeAfterCallingNext() + rightChild.calculateMaxReturnSize() - + rightChild.calculateRetainedSizeAfterCallingNext(); + + rightChild.calculateRetainedSizeAfterCallingNext() + + maxReturnSize; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Accumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Accumulator.java new file mode 100644 index 000000000000..734fa21c00cc --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Accumulator.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; + +public interface Accumulator { + long getEstimatedSize(); + + Accumulator copy(); + + void addInput(Column[] arguments); + + void addIntermediate(Column argument); + + void evaluateIntermediate(ColumnBuilder columnBuilder); + + void evaluateFinal(ColumnBuilder columnBuilder); + + void reset(); +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java new file mode 100644 index 000000000000..f5157f19543a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AccumulatorFactory.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation; + +import org.apache.iotdb.common.rpc.thrift.TAggregationType; +import org.apache.iotdb.db.queryengine.plan.expression.Expression; +import org.apache.iotdb.db.queryengine.plan.expression.binary.CompareBinaryExpression; +import org.apache.iotdb.db.queryengine.plan.expression.leaf.ConstantOperand; + +import org.apache.tsfile.enums.TSDataType; + +import java.util.List; +import java.util.Map; + +import static com.google.common.base.Preconditions.checkState; + +public class AccumulatorFactory { + + public static Accumulator createAccumulator( + String functionName, + TAggregationType aggregationType, + List inputDataTypes, + List inputExpressions, + Map inputAttributes, + boolean ascending) { + if (aggregationType == TAggregationType.UDAF) { + // If UDAF accumulator receives raw input, it needs to check input's attribute + throw new UnsupportedOperationException(); + } else { + return createBuiltinAccumulator( + aggregationType, inputDataTypes, inputExpressions, inputAttributes, ascending); + } + } + + public static Accumulator createBuiltinAccumulator( + TAggregationType aggregationType, + List inputDataTypes, + List inputExpressions, + Map inputAttributes, + boolean ascending) { + return isMultiInputAggregation(aggregationType) + ? createBuiltinMultiInputAccumulator(aggregationType, inputDataTypes) + : createBuiltinSingleInputAccumulator( + aggregationType, inputDataTypes.get(0), inputExpressions, inputAttributes, ascending); + } + + public static boolean isMultiInputAggregation(TAggregationType aggregationType) { + switch (aggregationType) { + case MAX_BY: + case MIN_BY: + return true; + default: + return false; + } + } + + public static Accumulator createBuiltinMultiInputAccumulator( + TAggregationType aggregationType, List inputDataTypes) { + switch (aggregationType) { + case MAX_BY: + checkState(inputDataTypes.size() == 2, "Wrong inputDataTypes size."); + // return new MaxByAccumulator(inputDataTypes.get(0), inputDataTypes.get(1)); + case MIN_BY: + checkState(inputDataTypes.size() == 2, "Wrong inputDataTypes size."); + // return new MinByAccumulator(inputDataTypes.get(0), inputDataTypes.get(1)); + default: + throw new IllegalArgumentException("Invalid Aggregation function: " + aggregationType); + } + } + + private static Accumulator createBuiltinSingleInputAccumulator( + TAggregationType aggregationType, + TSDataType tsDataType, + List inputExpressions, + Map inputAttributes, + boolean ascending) { + switch (aggregationType) { + case COUNT: + return new CountAccumulator(); + case AVG: + return new AvgAccumulator(tsDataType); + /*case SUM: + return new SumAccumulator(tsDataType); + case EXTREME: + return new ExtremeAccumulator(tsDataType); + case MAX_TIME: + return ascending ? new MaxTimeAccumulator() : new MaxTimeDescAccumulator(); + case MIN_TIME: + return ascending ? new MinTimeAccumulator() : new MinTimeDescAccumulator(); + case MAX_VALUE: + return new MaxValueAccumulator(tsDataType); + case MIN_VALUE: + return new MinValueAccumulator(tsDataType); + case LAST_VALUE: + return ascending + ? new LastValueAccumulator(tsDataType) + : new LastValueDescAccumulator(tsDataType); + case FIRST_VALUE: + return ascending + ? new FirstValueAccumulator(tsDataType) + : new FirstValueDescAccumulator(tsDataType); + case COUNT_IF: + return new CountIfAccumulator( + initKeepEvaluator(inputExpressions.get(1)), + Boolean.parseBoolean(inputAttributes.getOrDefault("ignoreNull", "true"))); + case TIME_DURATION: + return new TimeDurationAccumulator(); + case MODE: + return createModeAccumulator(tsDataType); + case COUNT_TIME: + return new CountTimeAccumulator(); + case STDDEV: + case STDDEV_SAMP: + return new VarianceAccumulator(tsDataType, VarianceAccumulator.VarianceType.STDDEV_SAMP); + case STDDEV_POP: + return new VarianceAccumulator(tsDataType, VarianceAccumulator.VarianceType.STDDEV_POP); + case VARIANCE: + case VAR_SAMP: + return new VarianceAccumulator(tsDataType, VarianceAccumulator.VarianceType.VAR_SAMP); + case VAR_POP: + return new VarianceAccumulator(tsDataType, VarianceAccumulator.VarianceType.VAR_POP);*/ + default: + throw new IllegalArgumentException("Invalid Aggregation function: " + aggregationType); + } + } + + /*private Accumulator createModeAccumulator(TSDataType tsDataType) { + switch (tsDataType) { + case BOOLEAN: + return new BooleanModeAccumulator(); + case TEXT: + return new BinaryModeAccumulator(); + case INT32: + return new IntModeAccumulator(); + case INT64: + return new LongModeAccumulator(); + case FLOAT: + return new FloatModeAccumulator(); + case DOUBLE: + return new DoubleModeAccumulator(); + case BLOB: + case STRING: + case TIMESTAMP: + case DATE: + default: + throw new IllegalArgumentException("Unknown data type: " + tsDataType); + } + }*/ + + @FunctionalInterface + public interface KeepEvaluator { + boolean apply(long keep); + } + + public static KeepEvaluator initKeepEvaluator(Expression keepExpression) { + // We have checked semantic in FE, + // keep expression must be ConstantOperand or CompareBinaryExpression here + if (keepExpression instanceof ConstantOperand) { + return keep -> keep >= Long.parseLong(keepExpression.getExpressionString()); + } else { + long constant = + Long.parseLong( + ((CompareBinaryExpression) keepExpression) + .getRightExpression() + .getExpressionString()); + switch (keepExpression.getExpressionType()) { + case LESS_THAN: + return keep -> keep < constant; + case LESS_EQUAL: + return keep -> keep <= constant; + case GREATER_THAN: + return keep -> keep > constant; + case GREATER_EQUAL: + return keep -> keep >= constant; + case EQUAL_TO: + return keep -> keep == constant; + case NON_EQUAL: + return keep -> keep != constant; + default: + throw new IllegalArgumentException( + "unsupported expression type: " + keepExpression.getExpressionType()); + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AggregationOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AggregationOperator.java new file mode 100644 index 000000000000..425f05f78675 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AggregationOperator.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation; + +import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; +import org.apache.iotdb.db.queryengine.execution.operator.Operator; +import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.execution.operator.process.ProcessOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator; +import org.apache.iotdb.db.queryengine.plan.planner.memory.MemoryReservationManager; + +import com.google.common.util.concurrent.ListenableFuture; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.read.common.block.column.RunLengthEncodedColumn; +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +public class AggregationOperator implements ProcessOperator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(AggregationOperator.class); + + private final OperatorContext operatorContext; + + private final Operator child; + + private final List aggregators; + + private final TsBlockBuilder resultBuilder; + + private final ColumnBuilder[] resultColumnsBuilder; + + private final long maxReturnSize = + TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); + + protected MemoryReservationManager memoryReservationManager; + + private boolean finished = false; + + public AggregationOperator( + OperatorContext operatorContext, Operator child, List aggregators) { + this.operatorContext = operatorContext; + this.child = child; + this.aggregators = aggregators; + this.resultBuilder = + new TsBlockBuilder( + aggregators.stream().map(Aggregator::getType).collect(toImmutableList())); + this.resultColumnsBuilder = resultBuilder.getValueColumnBuilders(); + this.memoryReservationManager = + operatorContext + .getDriverContext() + .getFragmentInstanceContext() + .getMemoryReservationContext(); + } + + @Override + public ListenableFuture isBlocked() { + return child.isBlocked(); + } + + @Override + public boolean hasNext() throws Exception { + return !finished; + } + + @Override + public TsBlock next() throws Exception { + // Each call only calculate at most once, no need to check time slice. + TsBlock block; + if (child.hasNextWithTimer()) { + block = child.nextWithTimer(); + if (block == null) { + return null; + } + + for (Aggregator aggregator : aggregators) { + aggregator.processBlock(block); + } + + return null; + } else { + // evaluate output + Column[] valueColumns = new Column[resultColumnsBuilder.length]; + for (int i = 0; i < aggregators.size(); i++) { + aggregators.get(i).evaluate(resultColumnsBuilder[i]); + valueColumns[i] = resultColumnsBuilder[i].build(); + } + + finished = true; + return TsBlock.wrapBlocksWithoutCopy( + 1, new RunLengthEncodedColumn(TableScanOperator.TIME_COLUMN_TEMPLATE, 1), valueColumns); + } + } + + @Override + public boolean isFinished() throws Exception { + return finished; + } + + @Override + public void close() throws Exception { + child.close(); + } + + @Override + public OperatorContext getOperatorContext() { + return operatorContext; + } + + @Override + public long calculateMaxPeekMemory() { + return Math.max( + child.calculateMaxPeekMemoryWithCounter(), + calculateRetainedSizeAfterCallingNext() + calculateMaxReturnSize()); + } + + @Override + public long calculateMaxReturnSize() { + return maxReturnSize; + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + return child.calculateMaxReturnSize() + child.calculateRetainedSizeAfterCallingNext(); + } + + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(child) + + aggregators.stream().mapToLong(Aggregator::getEstimatedSize).count() + + MemoryEstimationHelper.getEstimatedSizeOfAccountableObject(operatorContext) + + resultBuilder.getRetainedSizeInBytes(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Aggregator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Aggregator.java new file mode 100644 index 000000000000..cbd465efd8e8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/Aggregator.java @@ -0,0 +1,81 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; + +import com.google.common.primitives.Ints; +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; + +import java.util.List; +import java.util.OptionalInt; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class Aggregator { + private final Accumulator accumulator; + private final AggregationNode.Step step; + private final TSDataType outputType; + private final int[] inputChannels; + private final OptionalInt maskChannel; + + public Aggregator( + Accumulator accumulator, + AggregationNode.Step step, + TSDataType outputType, + List inputChannels, + OptionalInt maskChannel) { + this.accumulator = requireNonNull(accumulator, "accumulator is null"); + this.step = requireNonNull(step, "step is null"); + this.outputType = requireNonNull(outputType, "intermediateType is null"); + this.inputChannels = Ints.toArray(requireNonNull(inputChannels, "inputChannels is null")); + this.maskChannel = requireNonNull(maskChannel, "maskChannel is null"); + checkArgument( + step.isInputRaw() || inputChannels.size() == 1, + "expected 1 input channel for intermediate aggregation"); + } + + public TSDataType getType() { + return outputType; + } + + public void processBlock(TsBlock block) { + if (step.isInputRaw()) { + Column[] arguments = block.getColumns(inputChannels); + accumulator.addInput(arguments); + } else { + accumulator.addIntermediate(block.getColumn(inputChannels[0])); + } + } + + public void evaluate(ColumnBuilder columnBuilder) { + if (step.isOutputPartial()) { + accumulator.evaluateIntermediate(columnBuilder); + } else { + accumulator.evaluateFinal(columnBuilder); + } + } + + public void reset() { + accumulator.reset(); + } + + public long getEstimatedSize() { + return accumulator.getEstimatedSize(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java new file mode 100644 index 000000000000..d5c9eff3630b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/AvgAccumulator.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.BinaryColumnBuilder; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.write.UnSupportedDataTypeException; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; + +import static com.google.common.base.Preconditions.checkArgument; + +public class AvgAccumulator implements Accumulator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(AvgAccumulator.class); + private final TSDataType argumentDataType; + private long countValue; + private double sumValue; + private boolean initResult = false; + + public AvgAccumulator(TSDataType argumentDataType) { + this.argumentDataType = argumentDataType; + } + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE; + } + + @Override + public Accumulator copy() { + return new AvgAccumulator(argumentDataType); + } + + @Override + public void addInput(Column[] arguments) { + checkArgument(arguments.length == 1, "argument of Avg should be one column"); + switch (argumentDataType) { + case INT32: + addIntInput(arguments[0]); + return; + case INT64: + addLongInput(arguments[0]); + return; + case FLOAT: + addFloatInput(arguments[0]); + return; + case DOUBLE: + addDoubleInput(arguments[0]); + return; + case TEXT: + case BLOB: + case STRING: + case BOOLEAN: + case DATE: + case TIMESTAMP: + default: + throw new UnSupportedDataTypeException( + String.format("Unsupported data type in aggregation AVG : %s", argumentDataType)); + } + } + + @Override + public void addIntermediate(Column argument) { + checkArgument( + argument instanceof BinaryColumn, + "intermediate input and output of Avg should be BinaryColumn"); + if (argument.isNull(0)) { + return; + } + initResult = true; + deserialize(argument.getBinary(0).getValues()); + if (countValue == 0) { + initResult = false; + } + } + + @Override + public void evaluateIntermediate(ColumnBuilder columnBuilder) { + checkArgument( + columnBuilder instanceof BinaryColumnBuilder, + "intermediate input and output of Avg should be BinaryColumn"); + if (!initResult) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeBinary(new Binary(serializeState())); + } + } + + private void deserialize(byte[] bytes) { + countValue = BytesUtils.bytesToLong(bytes, Long.BYTES); + sumValue = BytesUtils.bytesToDouble(bytes, Long.BYTES); + } + + private byte[] serializeState() { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); + try { + dataOutputStream.writeLong(countValue); + dataOutputStream.writeDouble(sumValue); + } catch (IOException e) { + throw new UnsupportedOperationException( + "Failed to serialize intermediate result for AvgAccumulator.", e); + } + return byteArrayOutputStream.toByteArray(); + } + + private void addIntInput(Column column) { + int count = column.getPositionCount(); + for (int i = 0; i < count; i++) { + if (!column.isNull(i)) { + initResult = true; + countValue++; + sumValue += column.getInt(i); + } + } + } + + private void addLongInput(Column column) { + int count = column.getPositionCount(); + for (int i = 0; i < count; i++) { + if (!column.isNull(i)) { + initResult = true; + countValue++; + sumValue += column.getLong(i); + } + } + } + + private void addFloatInput(Column column) { + int count = column.getPositionCount(); + for (int i = 0; i < count; i++) { + if (!column.isNull(i)) { + initResult = true; + countValue++; + sumValue += column.getFloat(i); + } + } + } + + private void addDoubleInput(Column column) { + int count = column.getPositionCount(); + for (int i = 0; i < count; i++) { + if (!column.isNull(i)) { + initResult = true; + countValue++; + sumValue += column.getDouble(i); + } + } + } + + @Override + public void evaluateFinal(ColumnBuilder columnBuilder) { + if (!initResult) { + columnBuilder.appendNull(); + } else { + columnBuilder.writeDouble(sumValue / countValue); + } + } + + @Override + public void reset() { + initResult = false; + this.countValue = 0; + this.sumValue = 0.0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountAccumulator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountAccumulator.java new file mode 100644 index 000000000000..3f8765643766 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/aggregation/CountAccumulator.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.utils.RamUsageEstimator; + +import static com.google.common.base.Preconditions.checkArgument; + +public class CountAccumulator implements Accumulator { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(CountAccumulator.class); + private long countState = 0; + + @Override + public long getEstimatedSize() { + return INSTANCE_SIZE; + } + + @Override + public Accumulator copy() { + return new CountAccumulator(); + } + + @Override + public void addInput(Column[] arguments) { + checkArgument(arguments.length == 1, "argument of Count should be one column"); + int count = arguments[0].getPositionCount(); + if (!arguments[0].mayHaveNull()) { + countState += count; + } else { + for (int i = 0; i < count; i++) { + if (!arguments[0].isNull(i)) { + countState++; + } + } + } + } + + @Override + public void addIntermediate(Column argument) { + checkArgument(argument.getPositionCount() == 1, "partialResult should always be one line"); + if (argument.isNull(0)) { + return; + } + countState += argument.getLong(0); + } + + @Override + public void evaluateIntermediate(ColumnBuilder columnBuilder) { + columnBuilder.writeLong(countState); + } + + @Override + public void evaluateFinal(ColumnBuilder columnBuilder) { + columnBuilder.writeLong(countState); + } + + @Override + public void reset() { + countState = 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java index f6b8099928f3..5c449af0790f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/relational/ColumnTransformerBuilder.java @@ -79,6 +79,7 @@ import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.LeafColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.NullColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.leaf.TimeColumnTransformer; +import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.CoalesceColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.InBinaryMultiColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.InBooleanMultiColumnTransformer; import org.apache.iotdb.db.queryengine.transformation.dag.column.multi.InDoubleMultiColumnTransformer; @@ -1283,7 +1284,27 @@ protected ColumnTransformer visitSymbolReference(SymbolReference node, Context c @Override protected ColumnTransformer visitCoalesceExpression(CoalesceExpression node, Context context) { - throw new UnsupportedOperationException(String.format(UNSUPPORTED_EXPRESSION, node)); + if (!context.cache.containsKey(node)) { + if (context.hasSeen.containsKey(node)) { + ColumnTransformer columnTransformer = context.hasSeen.get(node); + IdentityColumnTransformer identity = + new IdentityColumnTransformer( + columnTransformer.getType(), + context.originSize + context.commonTransformerList.size()); + columnTransformer.addReferenceCount(); + context.commonTransformerList.add(columnTransformer); + context.leafList.add(identity); + context.inputDataTypes.add(getTSDataType(columnTransformer.getType())); + context.cache.put(node, identity); + } else { + List children = + node.getChildren().stream().map(c -> process(c, context)).collect(Collectors.toList()); + context.cache.put(node, new CoalesceColumnTransformer(children.get(0).getType(), children)); + } + } + ColumnTransformer res = context.cache.get(node); + res.addReferenceCount(); + return res; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java index a86dfe1dacfd..3ac30f633a7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/Coordinator.java @@ -54,8 +54,10 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowAINodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowConfigNodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB; @@ -329,8 +331,10 @@ private IQueryExecution createQueryExecutionForTableModel( || statement instanceof ShowRegions || statement instanceof ShowDataNodes || statement instanceof ShowConfigNodes + || statement instanceof ShowAINodes || statement instanceof Flush - || statement instanceof SetConfiguration) { + || statement instanceof SetConfiguration + || statement instanceof PipeStatement) { return new ConfigExecution( queryContext, null, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java index 11e8e42ec832..1f8905cc666f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java @@ -414,6 +414,7 @@ public void setSchemaTree(ISchemaTree schemaTree) { this.schemaTree = schemaTree; } + @Override public List getRedirectNodeList() { return redirectNodeList; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java index 05c3638a95d0..e03f75db230f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java @@ -66,6 +66,8 @@ default void setRealStatement(Statement realStatement) {} DataPartition getDataPartitionInfo(); + List getRedirectNodeList(); + void setRedirectNodeList(List redirectNodeList); void addEndPointToRedirectNodeList(TEndPoint endPoint); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/DataNodeSchemaLockManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/DataNodeSchemaLockManager.java index 598124ce0dec..11478cf1f77f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/DataNodeSchemaLockManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/lock/DataNodeSchemaLockManager.java @@ -21,7 +21,6 @@ import org.apache.iotdb.db.queryengine.common.MPPQueryContext; -import java.util.Map; import java.util.concurrent.locks.ReentrantReadWriteLock; public class DataNodeSchemaLockManager { @@ -45,18 +44,16 @@ private DataNodeSchemaLockManager() { } public void takeReadLock(final MPPQueryContext context, final SchemaLockType lockType) { - locks[lockType.ordinal()].readLock().lock(); - context.addAcquiredLockNum(lockType); + if (context.addAcquiredLock(lockType)) { + locks[lockType.ordinal()].readLock().lock(); + } } public void releaseReadLock(final MPPQueryContext queryContext) { - if (queryContext != null && !queryContext.getAcquiredLockNumMap().isEmpty()) { - final Map lockMap = queryContext.getAcquiredLockNumMap(); - for (final Map.Entry entry : lockMap.entrySet()) { - for (int i = 0; i < entry.getValue(); i++) { - locks[entry.getKey().ordinal()].readLock().unlock(); - } - } + if (queryContext != null && !queryContext.getAcquiredLocks().isEmpty()) { + queryContext + .getAcquiredLocks() + .forEach(lockType -> locks[lockType.ordinal()].readLock().unlock()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 2f5137bc5cdc..432a5111097b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -28,7 +28,10 @@ import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.CreatePipePluginTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.DropPipePluginTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowClusterTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowPipePluginsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableSetPropertiesTask; @@ -36,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DescribeTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DropDBTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowAINodesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowConfigNodesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowDataNodesTask; @@ -43,17 +47,28 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.UseDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.FlushTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetConfigurationTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.AlterPipeTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.CreatePipeTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.DropPipeTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.ShowPipeTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.StartPipeTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.pipe.StopPipeTask; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AddColumn; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ColumnDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipe; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipePlugin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentDatabase; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DescribeTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipe; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipePlugin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Flush; @@ -64,12 +79,17 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowAINodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowConfigNodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDataNodes; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipePlugins; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowRegions; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTables; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartPipe; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; import org.apache.iotdb.db.queryengine.plan.relational.type.TypeNotFoundException; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowClusterStatement; @@ -224,24 +244,32 @@ protected IConfigTask visitShowRegions( context.setQueryType(QueryType.READ); // As the implementation is identical, we'll simply translate to the // corresponding tree-model variant and execute that. - ShowRegionStatement treeStatement = new ShowRegionStatement(); + final ShowRegionStatement treeStatement = new ShowRegionStatement(); treeStatement.setRegionType(showRegions.getRegionType()); treeStatement.setStorageGroups(showRegions.getDatabases()); treeStatement.setNodeIds(showRegions.getNodeIds()); - return new ShowRegionTask(treeStatement); + return new ShowRegionTask(treeStatement, true); } @Override protected IConfigTask visitShowDataNodes( final ShowDataNodes showDataNodesStatement, final MPPQueryContext context) { context.setQueryType(QueryType.READ); - return new ShowDataNodesTask(showDataNodesStatement); + return new ShowDataNodesTask(); } + @Override protected IConfigTask visitShowConfigNodes( final ShowConfigNodes showConfigNodesStatement, final MPPQueryContext context) { context.setQueryType(QueryType.READ); - return new ShowConfigNodesTask(showConfigNodesStatement); + return new ShowConfigNodesTask(); + } + + @Override + protected IConfigTask visitShowAINodes( + final ShowAINodes showAINodesStatement, final MPPQueryContext context) { + context.setQueryType(QueryType.READ); + return new ShowAINodesTask(); } @Override @@ -434,4 +462,58 @@ private int parseIntFromLiteral(final Object value, final String name) { } return (int) parsedValue; } + + @Override + protected IConfigTask visitCreatePipe(CreatePipe node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + return new CreatePipeTask(node); + } + + @Override + protected IConfigTask visitAlterPipe(AlterPipe node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + return new AlterPipeTask(node); + } + + @Override + protected IConfigTask visitDropPipe(DropPipe node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + return new DropPipeTask(node); + } + + @Override + protected IConfigTask visitStartPipe(StartPipe node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + return new StartPipeTask(node); + } + + @Override + protected IConfigTask visitStopPipe(StopPipe node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + return new StopPipeTask(node); + } + + @Override + protected IConfigTask visitShowPipes(ShowPipes node, MPPQueryContext context) { + context.setQueryType(QueryType.READ); + return new ShowPipeTask(node); + } + + @Override + protected IConfigTask visitCreatePipePlugin(CreatePipePlugin node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + return new CreatePipePluginTask(node); + } + + @Override + protected IConfigTask visitDropPipePlugin(DropPipePlugin node, MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + return new DropPipePluginTask(node); + } + + @Override + protected IConfigTask visitShowPipePlugins(ShowPipePlugins node, MPPQueryContext context) { + context.setQueryType(QueryType.READ); + return new ShowPipePluginsTask(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java index c39f6277fd31..4f959422d190 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java @@ -38,6 +38,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.GetTimeSlotListTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.MigrateRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.SetTTLTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowAINodesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowClusterDetailsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowClusterIdTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowClusterTask; @@ -54,7 +55,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.UnSetTTLTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.CreateModelTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.DropModelTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.ShowAINodesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.ShowModelsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.AlterSchemaTemplateTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.CreateSchemaTemplateTask; @@ -370,7 +370,7 @@ public IConfigTask visitShowPipePlugins( @Override public IConfigTask visitShowRegion( ShowRegionStatement showRegionStatement, MPPQueryContext context) { - return new ShowRegionTask(showRegionStatement); + return new ShowRegionTask(showRegionStatement, false); } @Override @@ -441,6 +441,12 @@ public IConfigTask visitShowConfigNodes( return new ShowConfigNodesTask(); } + @Override + public IConfigTask visitShowAINodes( + ShowAINodesStatement showAINodesStatement, MPPQueryContext context) { + return new ShowAINodesTask(); + } + @Override public IConfigTask visitShowPipes( ShowPipesStatement showPipesStatement, MPPQueryContext context) { @@ -612,10 +618,4 @@ public IConfigTask visitShowModels( ShowModelsStatement showModelsStatement, MPPQueryContext context) { return new ShowModelsTask(showModelsStatement.getModelName()); } - - @Override - public IConfigTask visitShowAINodes( - ShowAINodesStatement showAINodesStatement, MPPQueryContext context) { - return new ShowAINodesTask(showAINodesStatement); - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 59d4c172f606..b233a2fc6c56 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -86,6 +86,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TDropTriggerReq; +import org.apache.iotdb.confignode.rpc.thrift.TFetchTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetDatabaseReq; import org.apache.iotdb.confignode.rpc.thrift.TGetPipePluginTableResp; import org.apache.iotdb.confignode.rpc.thrift.TGetRegionIdReq; @@ -152,6 +153,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.GetRegionIdTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.GetSeriesSlotListTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.GetTimeSlotListTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowAINodesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowClusterDetailsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowClusterIdTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowClusterTask; @@ -164,7 +166,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowTTLTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowTriggersTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowVariablesTask; -import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.ShowAINodesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.ShowModelsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DescribeTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowDBTask; @@ -183,10 +184,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.AlterLogicalViewNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowConfigNodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDataNodes; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowRegions; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountTimeSlotListStatement; @@ -202,12 +200,10 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.MigrateRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.SetTTLStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowClusterStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDataNodesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTTLStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.ShowAINodesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.AlterPipeStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipePluginStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipeStatement; @@ -285,6 +281,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -1391,10 +1388,11 @@ public SettableFuture showTTL(ShowTTLStatement showTTLStatemen } @Override - public SettableFuture showRegion(ShowRegionStatement showRegionStatement) { - SettableFuture future = SettableFuture.create(); + public SettableFuture showRegion( + final ShowRegionStatement showRegionStatement, final boolean isTableModel) { + final SettableFuture future = SettableFuture.create(); TShowRegionResp showRegionResp = new TShowRegionResp(); - TShowRegionReq showRegionReq = new TShowRegionReq(); + final TShowRegionReq showRegionReq = new TShowRegionReq(); showRegionReq.setConsensusGroupType(showRegionStatement.getRegionType()); if (showRegionStatement.getStorageGroups() == null) { showRegionReq.setDatabases(null); @@ -1404,7 +1402,7 @@ public SettableFuture showRegion(ShowRegionStatement showRegio .map(PartialPath::getFullPath) .collect(Collectors.toList())); } - try (ConfigNodeClient client = + try (final ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { showRegionResp = client.showRegion(showRegionReq); if (showRegionResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { @@ -1413,7 +1411,7 @@ public SettableFuture showRegion(ShowRegionStatement showRegio showRegionResp.getStatus().message, showRegionResp.getStatus().code)); return future; } - } catch (ClientManagerException | TException e) { + } catch (final ClientManagerException | TException e) { future.setException(e); } @@ -1430,13 +1428,12 @@ public SettableFuture showRegion(ShowRegionStatement showRegio } // build TSBlock - ShowRegionTask.buildTSBlock(showRegionResp, future); + ShowRegionTask.buildTSBlock(showRegionResp, future, isTableModel); return future; } @Override - public SettableFuture showDataNodes( - final ShowDataNodesStatement showDataNodesStatement) { + public SettableFuture showDataNodes() { final SettableFuture future = SettableFuture.create(); TShowDataNodesResp showDataNodesResp = new TShowDataNodesResp(); try (final ConfigNodeClient client = @@ -1478,6 +1475,24 @@ public SettableFuture showConfigNodes() { return future; } + @Override + public SettableFuture showAINodes() { + SettableFuture future = SettableFuture.create(); + TShowAINodesResp resp = new TShowAINodesResp(); + try (ConfigNodeClient client = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + resp = client.showAINodes(); + if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + future.setException(new IoTDBException(resp.getStatus().message, resp.getStatus().code)); + return future; + } + } catch (ClientManagerException | TException e) { + future.setException(e); + } + ShowAINodesTask.buildTsBlock(resp, future); + return future; + } + @Override public SettableFuture createSchemaTemplate( final CreateSchemaTemplateStatement createSchemaTemplateStatement) { @@ -2763,24 +2778,6 @@ public SettableFuture showModels(String modelName) { return future; } - @Override - public SettableFuture showAINodes(ShowAINodesStatement showAINodesStatement) { - SettableFuture future = SettableFuture.create(); - TShowAINodesResp resp = new TShowAINodesResp(); - try (ConfigNodeClient client = - CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - resp = client.showAINodes(); - if (resp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - future.setException(new IoTDBException(resp.getStatus().message, resp.getStatus().code)); - return future; - } - } catch (ClientManagerException | TException e) { - future.setException(e); - } - ShowAINodesTask.buildTsBlock(resp, future); - return future; - } - @Override public SettableFuture setSpaceQuota( SetSpaceQuotaStatement setSpaceQuotaStatement) { @@ -2957,60 +2954,6 @@ public SettableFuture showCluster(ShowCluster showCluster) { return showCluster(treeStatement); } - @Override - public SettableFuture showRegions(ShowRegions showRegions) { - // As the implementation is identical, we'll simply translate to the - // corresponding tree-model variant and execute that. - ShowRegionStatement treeStatement = new ShowRegionStatement(); - treeStatement.setRegionType(showRegions.getRegionType()); - treeStatement.setStorageGroups(showRegions.getDatabases()); - treeStatement.setNodeIds(showRegions.getNodeIds()); - return showRegion(treeStatement); - } - - @Override - public SettableFuture showDataNodes(ShowDataNodes showDataNodes) { - SettableFuture future = SettableFuture.create(); - TShowDataNodesResp showDataNodesResp = new TShowDataNodesResp(); - try (ConfigNodeClient client = - CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - showDataNodesResp = client.showDataNodes(); - if (showDataNodesResp.getStatus().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - future.setException( - new IoTDBException( - showDataNodesResp.getStatus().message, showDataNodesResp.getStatus().code)); - return future; - } - } catch (ClientManagerException | TException e) { - future.setException(e); - } - // build TSBlock - ShowDataNodesTask.buildTSBlock(showDataNodesResp, future); - return future; - } - - @Override - public SettableFuture showConfigNodes(ShowConfigNodes showConfigNodes) { - SettableFuture future = SettableFuture.create(); - TShowConfigNodesResp showConfigNodesResp = new TShowConfigNodesResp(); - try (ConfigNodeClient client = - CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - showConfigNodesResp = client.showConfigNodes(); - if (showConfigNodesResp.getStatus().getCode() - != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - future.setException( - new IoTDBException( - showConfigNodesResp.getStatus().message, showConfigNodesResp.getStatus().code)); - return future; - } - } catch (ClientManagerException | TException e) { - future.setException(e); - } - // build TSBlock - ShowConfigNodesTask.buildTSBlock(showConfigNodesResp, future); - return future; - } - @Override public SettableFuture useDatabase(Use useDB, IClientSession clientSession) { SettableFuture future = SettableFuture.create(); @@ -3197,6 +3140,22 @@ public SettableFuture showTables(final String database) { return future; } + @Override + public TFetchTableResp fetchTables(final Map> fetchTableMap) { + try (final ConfigNodeClient configNodeClient = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + final TFetchTableResp fetchTableResp = configNodeClient.fetchTables(fetchTableMap); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != fetchTableResp.getStatus().getCode()) { + LOGGER.warn("Failed to fetchTables, status is {}.", fetchTableResp); + } + return fetchTableResp; + } catch (final Exception e) { + return new TFetchTableResp( + new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode()) + .setMessage(e.toString())); + } + } + @Override public SettableFuture alterTableAddColumn( final String database, @@ -3305,14 +3264,14 @@ public SettableFuture alterTableSetProperties( return future; } - public void handlePipeConfigClientExit(String clientId) { + public void handlePipeConfigClientExit(final String clientId) { try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TSStatus status = configNodeClient.handlePipeConfigClientExit(clientId); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != status.getCode()) { LOGGER.warn("Failed to handlePipeConfigClientExit, status is {}.", status); } - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn("Failed to handlePipeConfigClientExit.", e); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index 68b32f6ec47d..16e32688c35d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; +import org.apache.iotdb.confignode.rpc.thrift.TFetchTableResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.db.protocol.session.IClientSession; @@ -34,10 +35,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metadata.write.view.AlterLogicalViewNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowConfigNodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDataNodes; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowRegions; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountTimeSlotListStatement; @@ -53,12 +51,10 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.MigrateRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.SetTTLStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowClusterStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDataNodesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowRegionStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.ShowTTLStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.CreateModelStatement; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.ShowAINodesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.AlterPipeStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipePluginStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipeStatement; @@ -95,6 +91,7 @@ import java.util.List; import java.util.Map; +import java.util.Set; public interface IConfigTaskExecutor { @@ -156,12 +153,15 @@ public interface IConfigTaskExecutor { SettableFuture showTTL(ShowTTLStatement showTTLStatement); - SettableFuture showRegion(ShowRegionStatement showRegionStatement); + SettableFuture showRegion( + final ShowRegionStatement showRegionStatement, final boolean isTableModel); - SettableFuture showDataNodes(ShowDataNodesStatement showDataNodesStatement); + SettableFuture showDataNodes(); SettableFuture showConfigNodes(); + SettableFuture showAINodes(); + SettableFuture createSchemaTemplate( CreateSchemaTemplateStatement createSchemaTemplateStatement); @@ -265,8 +265,6 @@ SettableFuture createModel( SettableFuture showModels(String modelName); - SettableFuture showAINodes(ShowAINodesStatement showAINodesStatement); - TPipeTransferResp handleTransferConfigPlan(String clientId, TPipeTransferReq req); void handlePipeConfigClientExit(String clientId); @@ -277,12 +275,6 @@ SettableFuture createModel( SettableFuture showCluster(ShowCluster showCluster); - SettableFuture showRegions(ShowRegions showRegions); - - SettableFuture showDataNodes(ShowDataNodes showDataNodes); - - SettableFuture showConfigNodes(ShowConfigNodes showConfigNodes); - SettableFuture useDatabase(final Use useDB, final IClientSession clientSession); SettableFuture dropDatabase(final DropDB dropDB); @@ -297,6 +289,8 @@ SettableFuture createTable( SettableFuture showTables(final String database); + TFetchTableResp fetchTables(final Map> fetchTableMap); + SettableFuture alterTableAddColumn( final String database, final String tableName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/CreatePipePluginTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/CreatePipePluginTask.java index 13ba7a9a7edd..73d2f4c37760 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/CreatePipePluginTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/CreatePipePluginTask.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipePlugin; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipePluginStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -34,6 +35,15 @@ public CreatePipePluginTask(CreatePipePluginStatement createPipePluginStatement) this.createPipePluginStatement = createPipePluginStatement; } + public CreatePipePluginTask(CreatePipePlugin node) { + createPipePluginStatement = + new CreatePipePluginStatement( + node.getPluginName(), + node.hasIfNotExistsCondition(), + node.getClassName(), + node.getUriString()); + } + @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DropPipePluginTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DropPipePluginTask.java index eb57f958430e..c014809aad22 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DropPipePluginTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/DropPipePluginTask.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipePlugin; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.DropPipePluginStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -34,6 +35,12 @@ public DropPipePluginTask(DropPipePluginStatement dropPipePluginStatement) { this.dropPipePluginStatement = dropPipePluginStatement; } + public DropPipePluginTask(DropPipePlugin node) { + dropPipePluginStatement = new DropPipePluginStatement(); + dropPipePluginStatement.setPluginName(node.getPluginName()); + dropPipePluginStatement.setIfExists(node.hasIfExistsCondition()); + } + @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/ShowAINodesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowAINodesTask.java similarity index 90% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/ShowAINodesTask.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowAINodesTask.java index f63a43aa1a36..e0bdffff149d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/model/ShowAINodesTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowAINodesTask.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model; +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata; import org.apache.iotdb.confignode.rpc.thrift.TAINodeInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowAINodesResp; @@ -28,7 +28,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; -import org.apache.iotdb.db.queryengine.plan.statement.metadata.model.ShowAINodesStatement; import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.util.concurrent.ListenableFuture; @@ -42,16 +41,14 @@ public class ShowAINodesTask implements IConfigTask { - private final ShowAINodesStatement showAINodesStatement; - - public ShowAINodesTask(ShowAINodesStatement showAINodesStatement) { - this.showAINodesStatement = showAINodesStatement; + public ShowAINodesTask() { + // do nothing } @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.showAINodes(showAINodesStatement); + return configTaskExecutor.showAINodes(); } public static void buildTsBlock( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java index 2fb899a88535..0c6b19238ca2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java @@ -53,7 +53,7 @@ public ShowDataNodesTask(ShowDataNodesStatement showDataNodesStatement) { @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.showDataNodes(showDataNodesStatement); + return configTaskExecutor.showDataNodes(); } public static void buildTSBlock( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowRegionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowRegionTask.java index 57e8b01af903..3578cc4b26f0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowRegionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowRegionTask.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata; import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType; +import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.confignode.rpc.thrift.TRegionInfo; import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; @@ -47,26 +48,30 @@ public class ShowRegionTask implements IConfigTask { private final ShowRegionStatement showRegionStatement; + private final boolean isTableModel; - public ShowRegionTask(ShowRegionStatement showRegionStatement) { + public ShowRegionTask(final ShowRegionStatement showRegionStatement, final boolean isTableModel) { this.showRegionStatement = showRegionStatement; + this.isTableModel = isTableModel; } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.showRegion(showRegionStatement); + return configTaskExecutor.showRegion(showRegionStatement, isTableModel); } public static void buildTSBlock( - TShowRegionResp showRegionResp, SettableFuture future) { - List outputDataTypes = + final TShowRegionResp showRegionResp, + final SettableFuture future, + final boolean isTableModel) { + final List outputDataTypes = ColumnHeaderConstant.showRegionColumnHeaders.stream() .map(ColumnHeader::getColumnType) .collect(Collectors.toList()); - TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); + final TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); if (showRegionResp.getRegionInfoList() != null) { - for (TRegionInfo regionInfo : showRegionResp.getRegionInfoList()) { + for (final TRegionInfo regionInfo : showRegionResp.getRegionInfoList()) { builder.getTimeColumnBuilder().writeLong(0L); builder.getColumnBuilder(0).writeInt(regionInfo.getConsensusGroupId().getId()); if (regionInfo.getConsensusGroupId().getType().ordinal() @@ -84,7 +89,13 @@ public static void buildTSBlock( .getColumnBuilder(2) .writeBinary( BytesUtils.valueOf(regionInfo.getStatus() == null ? "" : regionInfo.getStatus())); - builder.getColumnBuilder(3).writeBinary(BytesUtils.valueOf(regionInfo.getDatabase())); + builder + .getColumnBuilder(3) + .writeBinary( + BytesUtils.valueOf( + isTableModel + ? PathUtils.unQualifyDatabaseName(regionInfo.getDatabase()) + : regionInfo.getDatabase())); builder.getColumnBuilder(4).writeInt(regionInfo.getSeriesSlots()); builder.getColumnBuilder(5).writeLong(regionInfo.getTimeSlots()); builder.getColumnBuilder(6).writeInt(regionInfo.getDataNodeId()); @@ -103,7 +114,7 @@ public static void buildTSBlock( builder.declarePosition(); } } - DatasetHeader datasetHeader = DatasetHeaderFactory.getShowRegionHeader(); + final DatasetHeader datasetHeader = DatasetHeaderFactory.getShowRegionHeader(); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowAINodesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowAINodesTask.java new file mode 100644 index 000000000000..20b88f8d1de7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowAINodesTask.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; +import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.rpc.TSStatusCode; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.utils.BytesUtils; + +import java.util.List; +import java.util.stream.Collectors; + +public class ShowAINodesTask implements IConfigTask { + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.showAINodes(); + } + + public static void buildTSBlock( + TShowDataNodesResp showDataNodesResp, SettableFuture future) { + List outputDataTypes = + ColumnHeaderConstant.showDataNodesColumnHeaders.stream() + .map(ColumnHeader::getColumnType) + .collect(Collectors.toList()); + TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); + if (showDataNodesResp.getDataNodesInfoList() != null) { + for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) { + builder.getTimeColumnBuilder().writeLong(0L); + builder.getColumnBuilder(0).writeInt(dataNodeInfo.getDataNodeId()); + builder + .getColumnBuilder(1) + .writeBinary( + BytesUtils.valueOf( + dataNodeInfo.getStatus() == null ? "" : dataNodeInfo.getStatus())); + + builder.getColumnBuilder(2).writeBinary(BytesUtils.valueOf(dataNodeInfo.getRpcAddresss())); + builder.getColumnBuilder(3).writeInt(dataNodeInfo.getRpcPort()); + builder.getColumnBuilder(4).writeInt(dataNodeInfo.getDataRegionNum()); + + builder.getColumnBuilder(5).writeInt(dataNodeInfo.getSchemaRegionNum()); + builder.declarePosition(); + } + } + DatasetHeader datasetHeader = DatasetHeaderFactory.getShowDataNodesHeader(); + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowConfigNodesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowConfigNodesTask.java index 5b0ac00068d6..dfd2180daa2f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowConfigNodesTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowConfigNodesTask.java @@ -28,7 +28,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowConfigNodes; import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.util.concurrent.ListenableFuture; @@ -42,16 +41,10 @@ public class ShowConfigNodesTask implements IConfigTask { - private final ShowConfigNodes showConfigNodes; - - public ShowConfigNodesTask(ShowConfigNodes showConfigNodes) { - this.showConfigNodes = showConfigNodes; - } - @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.showConfigNodes(showConfigNodes); + return configTaskExecutor.showConfigNodes(); } public static void buildTSBlock( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDataNodesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDataNodesTask.java index 1ea2533296f7..744fe2018dff 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDataNodesTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowDataNodesTask.java @@ -28,7 +28,6 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDataNodes; import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.util.concurrent.ListenableFuture; @@ -42,16 +41,10 @@ public class ShowDataNodesTask implements IConfigTask { - private final ShowDataNodes showDataNodes; - - public ShowDataNodesTask(ShowDataNodes showDataNodes) { - this.showDataNodes = showDataNodes; - } - @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.showDataNodes(showDataNodes); + return configTaskExecutor.showDataNodes(); } public static void buildTSBlock( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java index 104501ae23f8..a251863eef46 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java @@ -22,6 +22,8 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.AlterPipeStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -34,6 +36,18 @@ public AlterPipeTask(AlterPipeStatement alterPipeStatement) { this.alterPipeStatement = alterPipeStatement; } + public AlterPipeTask(AlterPipe node) { + alterPipeStatement = new AlterPipeStatement(StatementType.ALTER_PIPE); + alterPipeStatement.setPipeName(node.getPipeName()); + alterPipeStatement.setIfExists(node.hasIfExistsCondition()); + alterPipeStatement.setExtractorAttributes(node.getExtractorAttributes()); + alterPipeStatement.setProcessorAttributes(node.getProcessorAttributes()); + alterPipeStatement.setConnectorAttributes(node.getConnectorAttributes()); + alterPipeStatement.setReplaceAllExtractorAttributes(node.isReplaceAllExtractorAttributes()); + alterPipeStatement.setReplaceAllProcessorAttributes(node.isReplaceAllProcessorAttributes()); + alterPipeStatement.setReplaceAllConnectorAttributes(node.isReplaceAllConnectorAttributes()); + } + @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/CreatePipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/CreatePipeTask.java index f0213c7d4f7b..f8e9d00c62b4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/CreatePipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/CreatePipeTask.java @@ -22,6 +22,8 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipe; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.CreatePipeStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -34,6 +36,15 @@ public CreatePipeTask(CreatePipeStatement createPipeStatement) { this.createPipeStatement = createPipeStatement; } + public CreatePipeTask(CreatePipe createPipe) { + createPipeStatement = new CreatePipeStatement(StatementType.CREATE_PIPE); + createPipeStatement.setPipeName(createPipe.getPipeName()); + createPipeStatement.setIfNotExists(createPipe.hasIfNotExistsCondition()); + createPipeStatement.setExtractorAttributes(createPipe.getExtractorAttributes()); + createPipeStatement.setProcessorAttributes(createPipe.getProcessorAttributes()); + createPipeStatement.setConnectorAttributes(createPipe.getConnectorAttributes()); + } + @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java index f3449f369901..a28ca91f3c65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java @@ -22,6 +22,8 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipe; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.DropPipeStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -34,6 +36,12 @@ public DropPipeTask(DropPipeStatement dropPipeStatement) { this.dropPipeStatement = dropPipeStatement; } + public DropPipeTask(DropPipe node) { + dropPipeStatement = new DropPipeStatement(StatementType.DROP_PIPE); + dropPipeStatement.setPipeName(node.getPipeName()); + dropPipeStatement.setIfExists(node.hasIfExistsCondition()); + } + @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java index a4d0a9505b45..e4075aa847cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipes; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.ShowPipesStatement; import org.apache.iotdb.db.utils.DateTimeUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -51,6 +52,12 @@ public ShowPipeTask(final ShowPipesStatement showPipesStatement) { this.showPipesStatement = showPipesStatement; } + public ShowPipeTask(ShowPipes node) { + showPipesStatement = new ShowPipesStatement(); + showPipesStatement.setPipeName(node.getPipeName()); + showPipesStatement.setWhereClause(node.hasWhereClause()); + } + @Override public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java index 62b55bb017c9..39ff162a43c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java @@ -22,6 +22,8 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartPipe; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.StartPipeStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -34,6 +36,11 @@ public StartPipeTask(StartPipeStatement startPipeStatement) { this.startPipeStatement = startPipeStatement; } + public StartPipeTask(StartPipe node) { + startPipeStatement = new StartPipeStatement(StatementType.START_PIPE); + startPipeStatement.setPipeName(node.getPipeName()); + } + @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java index c18070a5027e..04806bef42f6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java @@ -22,6 +22,8 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopPipe; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.metadata.pipe.StopPipeStatement; import com.google.common.util.concurrent.ListenableFuture; @@ -34,6 +36,11 @@ public StopPipeTask(StopPipeStatement stopPipeStatement) { this.stopPipeStatement = stopPipeStatement; } + public StopPipeTask(StopPipe node) { + stopPipeStatement = new StopPipeStatement(StatementType.STOP_PIPE); + stopPipeStatement.setPipeName(node.getPipeName()); + } + @Override public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) throws InterruptedException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index f184d34dba05..6377a72c843d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -257,6 +257,7 @@ import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_RESULT_NODES; import static org.apache.iotdb.db.queryengine.plan.optimization.LimitOffsetPushDown.canPushDownLimitOffsetToGroupByTime; import static org.apache.iotdb.db.queryengine.plan.optimization.LimitOffsetPushDown.pushDownLimitOffsetToTimeParameter; +import static org.apache.iotdb.db.utils.TimestampPrecisionUtils.TIMESTAMP_PRECISION; import static org.apache.iotdb.db.utils.TimestampPrecisionUtils.currPrecision; import static org.apache.iotdb.db.utils.constant.SqlConstant.CAST_FUNCTION; import static org.apache.iotdb.db.utils.constant.SqlConstant.CAST_TYPE; @@ -1992,7 +1993,10 @@ private long parseTimeValue(ConstantContext constant) { return Long.parseLong(constant.INTEGER_LITERAL().getText()); } catch (NumberFormatException e) { throw new SemanticException( - String.format("Can not parse %s to long value", constant.INTEGER_LITERAL().getText())); + String.format( + "Current system timestamp precision is %s, " + + "please check whether the timestamp %s is correct.", + TIMESTAMP_PRECISION, constant.INTEGER_LITERAL().getText())); } } else if (constant.dateExpression() != null) { return parseDateExpression(constant.dateExpression(), CommonDateTimeUtils.currentTime()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java index 8e497ca056c9..20461c5011cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java @@ -530,11 +530,11 @@ public static InsertRowsOfOneDeviceStatement createStatement( return insertStatement; } - public static DatabaseSchemaStatement createStatement(String database) + public static DatabaseSchemaStatement createStatement(final String database) throws IllegalPathException { - long startTime = System.nanoTime(); + final long startTime = System.nanoTime(); // construct create database statement - DatabaseSchemaStatement statement = + final DatabaseSchemaStatement statement = new DatabaseSchemaStatement(DatabaseSchemaStatement.DatabaseSchemaStatementType.CREATE); statement.setDatabasePath(parseDatabaseRawString(database)); PERFORMANCE_OVERVIEW_METRICS.recordParseCost(System.nanoTime() - startTime); @@ -889,8 +889,9 @@ private static void addMeasurementAndValue( insertRowStatement.setMeasurements(newMeasurements.toArray(new String[0])); } - private static PartialPath parseDatabaseRawString(String database) throws IllegalPathException { - PartialPath databasePath = new PartialPath(database); + private static PartialPath parseDatabaseRawString(final String database) + throws IllegalPathException { + final PartialPath databasePath = new PartialPath(database); if (databasePath.getNodeLength() < 2) { throw new IllegalPathException(database); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index cf9a3dbba652..cf0036ffbade 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -1176,7 +1176,7 @@ public Operator visitAggregationMergeSort( .addOperatorContext( context.getNextOperatorId(), node.getPlanNodeId(), - TreeMergeSortOperator.class.getSimpleName()); + AggregationMergeSortOperator.class.getSimpleName()); List dataTypes = getOutputColumnTypes(node, context.getTypeProvider()); List children = dealWithConsumeAllChildrenPipelineBreaker(node, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java index 754cde579126..7a5cc0db0453 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/TableOperatorGenerator.java @@ -49,8 +49,12 @@ import org.apache.iotdb.db.queryengine.execution.operator.sink.IdentitySinkOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.ExchangeOperator; -import org.apache.iotdb.db.queryengine.execution.operator.source.relational.InnerJoinOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableFullOuterJoinOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableInnerJoinOperator; import org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Accumulator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.AggregationOperator; +import org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.Aggregator; import org.apache.iotdb.db.queryengine.execution.relational.ColumnTransformerBuilder; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -92,6 +96,8 @@ import com.google.common.collect.ImmutableMap; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.type.RowType; +import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.filter.basic.Filter; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -108,6 +114,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalInt; import java.util.Set; import java.util.stream.Collectors; @@ -117,8 +124,10 @@ import static org.apache.iotdb.db.queryengine.common.DataNodeEndPoints.isSameNode; import static org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.MergeSortComparator.getComparatorForTable; import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.TableScanOperator.constructAlignedPath; +import static org.apache.iotdb.db.queryengine.execution.operator.source.relational.aggregation.AccumulatorFactory.createAccumulator; import static org.apache.iotdb.db.queryengine.plan.analyze.PredicateUtils.convertPredicateToFilter; import static org.apache.iotdb.db.queryengine.plan.planner.OperatorTreeGenerator.ASC_TIME_COMPARATOR; +import static org.apache.iotdb.db.queryengine.plan.relational.metadata.TableBuiltinAggregationFunction.getAggregationTypeByFuncName; import static org.apache.iotdb.db.queryengine.plan.relational.type.InternalTypeManager.getTSDataType; /** This Visitor is responsible for transferring Table PlanNode Tree to Table Operator Tree. */ @@ -760,11 +769,15 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { Operator leftChild = node.getLeftChild().accept(this, context); Operator rightChild = node.getRightChild().accept(this, context); + int leftTimeColumnPosition = + node.getLeftChild().getOutputSymbols().indexOf(node.getCriteria().get(0).getLeft()); int[] leftOutputSymbolIdx = new int[node.getLeftOutputSymbols().size()]; for (int i = 0; i < leftOutputSymbolIdx.length; i++) { leftOutputSymbolIdx[i] = node.getLeftChild().getOutputSymbols().indexOf(node.getLeftOutputSymbols().get(i)); } + int rightTimeColumnPosition = + node.getRightChild().getOutputSymbols().indexOf(node.getCriteria().get(0).getRight()); int[] rightOutputSymbolIdx = new int[node.getRightOutputSymbols().size()]; for (int i = 0; i < rightOutputSymbolIdx.length; i++) { rightOutputSymbolIdx[i] = @@ -772,11 +785,24 @@ public Operator visitJoin(JoinNode node, LocalExecutionPlanContext context) { } if (requireNonNull(node.getJoinType()) == JoinNode.JoinType.INNER) { - return new InnerJoinOperator( + return new TableInnerJoinOperator( operatorContext, leftChild, + leftTimeColumnPosition, leftOutputSymbolIdx, rightChild, + rightTimeColumnPosition, + rightOutputSymbolIdx, + ASC_TIME_COMPARATOR, + dataTypes); + } else if (requireNonNull(node.getJoinType()) == JoinNode.JoinType.FULL) { + return new TableFullOuterJoinOperator( + operatorContext, + leftChild, + leftTimeColumnPosition, + leftOutputSymbolIdx, + rightChild, + rightTimeColumnPosition, rightOutputSymbolIdx, ASC_TIME_COMPARATOR, dataTypes); @@ -896,7 +922,86 @@ public Operator visitTableDeviceQueryCount( @Override public Operator visitAggregation(AggregationNode node, LocalExecutionPlanContext context) { - throw new UnsupportedOperationException("Agg-BE not supported"); + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + AggregationNode.class.getSimpleName()); + Operator child = node.getChild().accept(this, context); + + if (node.getGroupingKeys().isEmpty()) { + return planGlobalAggregation(node, child, context.getTypeProvider(), operatorContext); + } + + throw new UnsupportedOperationException(); + // return planGroupByAggregation(node, child, outputTypes, operatorContext); + } + + private Operator planGlobalAggregation( + AggregationNode node, Operator child, TypeProvider typeProvider, OperatorContext context) { + + Map aggregationMap = node.getAggregations(); + ImmutableList.Builder aggregatorBuilder = new ImmutableList.Builder<>(); + Map childLayout = + makeLayoutFromOutputSymbols(node.getChild().getOutputSymbols()); + + node.getOutputSymbols() + .forEach( + symbol -> + aggregatorBuilder.add( + buildAggregator( + childLayout, aggregationMap.get(symbol), node.getStep(), typeProvider))); + return new AggregationOperator(context, child, aggregatorBuilder.build()); + } + + private ImmutableMap makeLayoutFromOutputSymbols(List outputSymbols) { + ImmutableMap.Builder outputMappings = ImmutableMap.builder(); + int channel = 0; + for (Symbol symbol : outputSymbols) { + outputMappings.put(symbol, channel); + channel++; + } + return outputMappings.buildOrThrow(); + } + + private Aggregator buildAggregator( + Map childLayout, + AggregationNode.Aggregation aggregation, + AggregationNode.Step step, + TypeProvider typeProvider) { + List argumentChannels = new ArrayList<>(); + List argumentTypes = new ArrayList<>(); + for (Expression argument : aggregation.getArguments()) { + Symbol argumentSymbol = Symbol.from(argument); + argumentChannels.add(childLayout.get(argumentSymbol)); + + // get argument types + Type type = typeProvider.getTableModelType(argumentSymbol); + if (type instanceof RowType) { + type.getTypeParameters().forEach(subType -> argumentTypes.add(getTSDataType(subType))); + } else { + argumentTypes.add(getTSDataType(type)); + } + } + + String functionName = aggregation.getResolvedFunction().getSignature().getName(); + Accumulator accumulator = + createAccumulator( + functionName, + getAggregationTypeByFuncName(functionName), + argumentTypes, + Collections.emptyList(), + Collections.emptyMap(), + true); + + return new Aggregator( + accumulator, + step, + getTSDataType(aggregation.getResolvedFunction().getSignature().getReturnType()), + argumentChannels, + OptionalInt.empty()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index d40d1c50b840..3a2027d1927d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -248,7 +248,7 @@ private Map collectSplitRanges() { return deviceIDSplitInfoMap; } - private Map> splitByReplicaSet( + protected Map> splitByReplicaSet( Map deviceIDSplitInfoMap, IAnalysis analysis) { Map> splitMap = new HashMap<>(); @@ -1189,12 +1189,12 @@ public IDeviceID getDeviceID(int rowIdx) { return deviceID; } - private static class PartitionSplitInfo { + protected static class PartitionSplitInfo { // for each List in split, they are range1.start, range1.end, range2.start, range2.end, ... - private List ranges = new ArrayList<>(); - private List timePartitionSlots = new ArrayList<>(); - private List replicaSets; + List ranges = new ArrayList<>(); + List timePartitionSlots = new ArrayList<>(); + List replicaSets; } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java index c2129f8d8d73..b6b97457cff0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java @@ -19,10 +19,13 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node.write; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.db.exception.query.OutOfTTLException; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; @@ -41,7 +44,9 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.function.IntToLongFunction; public class RelationalInsertTabletNode extends InsertTabletNode { @@ -139,6 +144,44 @@ protected InsertTabletNode getEmptySplit(int count) { columnCategories); } + protected Map> splitByReplicaSet( + Map deviceIDSplitInfoMap, IAnalysis analysis) { + Map> splitMap = new HashMap<>(); + Map endPointMap = new HashMap<>(); + + for (Map.Entry entry : deviceIDSplitInfoMap.entrySet()) { + final IDeviceID deviceID = entry.getKey(); + final PartitionSplitInfo splitInfo = entry.getValue(); + final List replicaSets = + analysis + .getDataPartitionInfo() + .getDataRegionReplicaSetForWriting( + deviceID, splitInfo.timePartitionSlots, analysis.getDatabaseName()); + splitInfo.replicaSets = replicaSets; + // collect redirectInfo + endPointMap.put( + deviceID, + replicaSets + .get(replicaSets.size() - 1) + .getDataNodeLocations() + .get(0) + .getClientRpcEndPoint()); + for (int i = 0; i < replicaSets.size(); i++) { + List subRanges = + splitMap.computeIfAbsent(replicaSets.get(i), x -> new ArrayList<>()); + subRanges.add(splitInfo.ranges.get(2 * i)); + subRanges.add(splitInfo.ranges.get(2 * i + 1)); + } + } + List redirectNodeList = new ArrayList<>(times.length); + for (int i = 0; i < times.length; i++) { + IDeviceID deviceId = getDeviceID(i); + redirectNodeList.add(endPointMap.get(deviceId)); + } + analysis.setRedirectNodeList(redirectNodeList); + return splitMap; + } + public static RelationalInsertTabletNode deserialize(ByteBuffer byteBuffer) { RelationalInsertTabletNode insertNode = new RelationalInsertTabletNode(new PlanNodeId("")); insertNode.subDeserialize(byteBuffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index a9ddeeea37eb..3ae791bd4968 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -754,6 +754,11 @@ public void upsertDataPartition(DataPartition targetDataPartition) { } } + @Override + public List getRedirectNodeList() { + return redirectNodeList; + } + @Override public void setRedirectNodeList(List redirectNodeList) { this.redirectNodeList = redirectNodeList; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java index cb9e7090f8a8..9407de3ce473 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionTreeUtils.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer; -import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableBuiltinAggregationFunction; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DefaultExpressionTraversalVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@ -93,6 +93,7 @@ public static QualifiedName asQualifiedName(Expression expression) { static boolean isAggregationFunction(String functionName) { // TODO consider UDAF - return BuiltinAggregationFunction.getNativeFunctionNames().contains(functionName.toLowerCase()); + return TableBuiltinAggregationFunction.getNativeFunctionNames() + .contains(functionName.toLowerCase()); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index b74c57286aff..e098d231336a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -45,12 +45,15 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AliasedRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllColumns; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllRows; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateIndex; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateOrUpdateDevice; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipe; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipePlugin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Delete; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression; @@ -59,6 +62,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropFunction; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropIndex; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipe; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipePlugin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Except; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; @@ -102,11 +107,15 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowFunctions; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowIndex; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipePlugins; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTables; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; @@ -1886,10 +1895,7 @@ protected Scope visitJoin(Join node, Optional scope) { createAndAssignScope( node, scope, left.getRelationType().joinWith(right.getRelationType())); - if (node.getType() == Join.Type.CROSS - || node.getType() == LEFT - || node.getType() == RIGHT - || node.getType() == FULL) { + if (node.getType() == Join.Type.CROSS || node.getType() == LEFT || node.getType() == RIGHT) { throw new SemanticException( String.format( "%s JOIN is not supported, only support INNER JOIN in current version.", @@ -2738,6 +2744,51 @@ private Expression analyzeAndRewriteExpression( scope.getRelationType().getAllFields(); return translationMap.rewrite(expression); } + + @Override + protected Scope visitCreatePipe(CreatePipe node, Optional context) { + return createAndAssignScope(node, context); + } + + @Override + protected Scope visitAlterPipe(AlterPipe node, Optional context) { + return createAndAssignScope(node, context); + } + + @Override + protected Scope visitDropPipe(DropPipe node, Optional context) { + return createAndAssignScope(node, context); + } + + @Override + protected Scope visitStartPipe(StartPipe node, Optional context) { + return createAndAssignScope(node, context); + } + + @Override + protected Scope visitStopPipe(StopPipe node, Optional context) { + return createAndAssignScope(node, context); + } + + @Override + protected Scope visitShowPipes(ShowPipes node, Optional context) { + return createAndAssignScope(node, context); + } + + @Override + protected Scope visitCreatePipePlugin(CreatePipePlugin node, Optional context) { + return createAndAssignScope(node, context); + } + + @Override + protected Scope visitDropPipePlugin(DropPipePlugin node, Optional context) { + return createAndAssignScope(node, context); + } + + @Override + protected Scope visitShowPipePlugins(ShowPipePlugins node, Optional context) { + return createAndAssignScope(node, context); + } } private static boolean hasScopeAsLocalParent(Scope root, Scope parent) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java index cfabb03c6aa8..22e8acec47b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java @@ -185,7 +185,8 @@ DataPartition getDataPartitionWithUnclosedTimeRange( final String database, final List sgNameToQueryParamsMap); /** + * @param withTime some function with time can also use Statistics, like first_by, last_by * @return if the Aggregation can use statistics to optimize */ - boolean canUseStatistics(final String name); + boolean canUseStatistics(final String name, boolean withTime); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableBuiltinAggregationFunction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableBuiltinAggregationFunction.java index 8435938771a4..72f0cdadd026 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableBuiltinAggregationFunction.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableBuiltinAggregationFunction.java @@ -19,6 +19,8 @@ package org.apache.iotdb.db.queryengine.plan.relational.metadata; +import org.apache.iotdb.common.rpc.thrift.TAggregationType; + import com.google.common.collect.ImmutableList; import org.apache.tsfile.read.common.type.Type; @@ -65,8 +67,8 @@ public String getFunctionName() { private static final Set NATIVE_FUNCTION_NAMES = new HashSet<>( - Arrays.stream(org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction.values()) - .map(org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction::getFunctionName) + Arrays.stream(TableBuiltinAggregationFunction.values()) + .map(TableBuiltinAggregationFunction::getFunctionName) .collect(Collectors.toList())); public static Set getNativeFunctionNames() { @@ -76,7 +78,7 @@ public static Set getNativeFunctionNames() { /** * @return if the Aggregation can use statistics to optimize */ - public static boolean canUseStatistics(String name) { + public static boolean canUseStatistics(String name, boolean withTime) { final String functionName = name.toLowerCase(); switch (functionName) { case "sum": @@ -87,10 +89,10 @@ public static boolean canUseStatistics(String name) { case "min": case "first": case "last": - case "time_duration": return true; case "first_by": case "last_by": + return withTime; case "mode": case "max_by": case "min_by": @@ -106,11 +108,20 @@ public static boolean canUseStatistics(String name) { } } - public static List getIntermediateTypes(String name, Type originalType) { + public static List getIntermediateTypes(String name, List originalArgumentTypes) { if (AVG.functionName.equalsIgnoreCase(name)) { return ImmutableList.of(DOUBLE, INT32); } else { - return ImmutableList.of(originalType); + return ImmutableList.copyOf(originalArgumentTypes); + } + } + + public static TAggregationType getAggregationTypeByFuncName(String funcName) { + if (NATIVE_FUNCTION_NAMES.contains(funcName)) { + return TAggregationType.valueOf(funcName.toUpperCase()); + } else { + // fallback to UDAF if no enum found + return TAggregationType.UDAF; } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java index 7902b37d68ec..430401218455 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java @@ -524,12 +524,12 @@ && isIntegerNumber(argumentTypes.get(2)))) { // builtin aggregation function // check argument type - switch (functionName.toLowerCase()) { + switch (functionName.toLowerCase(Locale.ENGLISH)) { case SqlConstant.AVG: case SqlConstant.SUM: case SqlConstant.EXTREME: - case SqlConstant.MIN_VALUE: - case SqlConstant.MAX_VALUE: + case SqlConstant.MIN: + case SqlConstant.MAX: case SqlConstant.STDDEV: case SqlConstant.STDDEV_POP: case SqlConstant.STDDEV_SAMP: @@ -543,11 +543,6 @@ && isIntegerNumber(argumentTypes.get(2)))) { functionName)); } break; - case SqlConstant.MIN_TIME: - case SqlConstant.MAX_TIME: - case SqlConstant.FIRST_VALUE: - case SqlConstant.LAST_VALUE: - case SqlConstant.TIME_DURATION: case SqlConstant.MODE: if (argumentTypes.size() != 1) { throw new SemanticException( @@ -555,6 +550,19 @@ && isIntegerNumber(argumentTypes.get(2)))) { "Aggregate functions [%s] should only have one argument", functionName)); } break; + case SqlConstant.FIRST: + case SqlConstant.LAST: + if (argumentTypes.size() != 2) { + throw new SemanticException( + String.format( + "Aggregate functions [%s] should only have two arguments", functionName)); + } else if (!isTimestampType(argumentTypes.get(1))) { + throw new SemanticException( + String.format( + "Second argument of Aggregate functions [%s] should be orderable", functionName)); + } + case SqlConstant.FIRST_BY: + case SqlConstant.LAST_BY: case SqlConstant.MAX_BY: case SqlConstant.MIN_BY: if (argumentTypes.size() != 2) { @@ -575,18 +583,17 @@ && isIntegerNumber(argumentTypes.get(2)))) { } // get return type - switch (functionName.toLowerCase()) { - case SqlConstant.MIN_TIME: - case SqlConstant.MAX_TIME: + switch (functionName.toLowerCase(Locale.ENGLISH)) { case SqlConstant.COUNT: - case SqlConstant.TIME_DURATION: return INT64; - case SqlConstant.MIN_VALUE: - case SqlConstant.LAST_VALUE: - case SqlConstant.FIRST_VALUE: - case SqlConstant.MAX_VALUE: + case SqlConstant.FIRST: + case SqlConstant.LAST: + case SqlConstant.FIRST_BY: + case SqlConstant.LAST_BY: case SqlConstant.EXTREME: case SqlConstant.MODE: + case SqlConstant.MAX: + case SqlConstant.MIN: case SqlConstant.MAX_BY: case SqlConstant.MIN_BY: return argumentTypes.get(0); @@ -698,8 +705,8 @@ public DataPartition getDataPartitionWithUnclosedTimeRange( } @Override - public boolean canUseStatistics(String functionName) { - return TableBuiltinAggregationFunction.canUseStatistics(functionName); + public boolean canUseStatistics(String functionName, boolean withTime) { + return TableBuiltinAggregationFunction.canUseStatistics(functionName, withTime); } public static boolean isTwoNumericType(List argumentTypes) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java index 48c95f8c4692..d9e7663d19b9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/TableHeaderSchemaValidator.java @@ -52,6 +52,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -80,19 +81,25 @@ public static TableHeaderSchemaValidator getInstance() { } public Optional validateTableHeaderSchema( - String database, TableSchema tableSchema, MPPQueryContext context, boolean allowCreateTable) { + final String database, + final TableSchema tableSchema, + final MPPQueryContext context, + final boolean allowCreateTable) { // The schema cache R/W and fetch operation must be locked together thus the cache clean // operation executed by delete timeSeries will be effective. DataNodeSchemaLockManager.getInstance() .takeReadLock(context, SchemaLockType.VALIDATE_VS_DELETION); - List inputColumnList = tableSchema.getColumns(); + final List inputColumnList = tableSchema.getColumns(); if (inputColumnList == null || inputColumnList.isEmpty()) { throw new IllegalArgumentException( "No column other than Time present, please check the request"); } - TsTable table = DataNodeTableCache.getInstance().getTable(database, tableSchema.getTableName()); - List missingColumnList = new ArrayList<>(); + // Get directly if there is a table because we do not want "addColumn" to affect + // original writings + TsTable table = + DataNodeTableCache.getInstance().getTableInWrite(database, tableSchema.getTableName()); + final List missingColumnList = new ArrayList<>(); // first round validate, check existing schema if (table == null) { @@ -112,25 +119,36 @@ public Optional validateTableHeaderSchema( } } - for (ColumnSchema columnSchema : inputColumnList) { + boolean refreshed = false; + for (final ColumnSchema columnSchema : inputColumnList) { TsTableColumnSchema existingColumn = table.getColumnSchema(columnSchema.getName()); - if (existingColumn == null) { - // check arguments for column auto creation - if (columnSchema.getColumnCategory() == null) { - throw new SemanticException( - String.format( - "Unknown column category for %s. Cannot auto create column.", - columnSchema.getName()), - TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode()); + if (Objects.isNull(existingColumn)) { + if (!refreshed) { + // Refresh because there may be new columns added and failed to commit + // Allow refresh only once to avoid too much failure columns in sql when there are column + // procedures + refreshed = true; + table = DataNodeTableCache.getInstance().getTable(database, tableSchema.getTableName()); + existingColumn = table.getColumnSchema(columnSchema.getName()); } - if (columnSchema.getType() == null) { - throw new SemanticException( - String.format( - "Unknown column data type for %s. Cannot auto create column.", - columnSchema.getName()), - TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode()); + if (Objects.isNull(existingColumn)) { + // check arguments for column auto creation + if (columnSchema.getColumnCategory() == null) { + throw new SemanticException( + String.format( + "Unknown column category for %s. Cannot auto create column.", + columnSchema.getName()), + TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode()); + } + if (columnSchema.getType() == null) { + throw new SemanticException( + String.format( + "Unknown column data type for %s. Cannot auto create column.", + columnSchema.getName()), + TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode()); + } + missingColumnList.add(columnSchema); } - missingColumnList.add(columnSchema); } else { // leave measurement columns' dataType checking to the caller, then the caller can decide // whether to do partial insert @@ -145,7 +163,7 @@ public Optional validateTableHeaderSchema( } } - List resultColumnList = new ArrayList<>(); + final List resultColumnList = new ArrayList<>(); if (!missingColumnList.isEmpty() && IoTDBDescriptor.getInstance().getConfig().isAutoCreateSchemaEnabled()) { // TODO table metadata: authority check for table alter diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java index 37bbc69c0603..c587774bcef8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AliasedRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CoalesceExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Except; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@ -83,6 +84,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.QueryPlanner.coerceIfNecessary; import static org.apache.iotdb.db.queryengine.plan.relational.planner.ir.IrUtils.extractPredicates; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join.Type.CROSS; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join.Type.FULL; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join.Type.IMPLICIT; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join.Type.INNER; @@ -329,12 +331,15 @@ If casts are redundant (due to column type and common type being equal), Symbol output = symbolAllocator.newSymbol(column, analysis.getType(column)); outputs.add(output); queryContext.getTypeProvider().putTableModelType(output, LongType.INT64); - assignments.put( - output, leftJoinColumns.get(column).toSymbolReference() - // new CoalesceExpression( - // leftJoinColumns.get(column).toSymbolReference(), - // rightJoinColumns.get(column).toSymbolReference()) - ); + if (node.getType() == INNER) { + assignments.put(output, leftJoinColumns.get(column).toSymbolReference()); + } else if (node.getType() == FULL) { + assignments.put( + output, + new CoalesceExpression( + leftJoinColumns.get(column).toSymbolReference(), + rightJoinColumns.get(column).toSymbolReference())); + } } for (int field : joinAnalysis.getOtherLeftFields()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java index e750e2d95a8a..1b90384efc31 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/TableModelPlanner.java @@ -38,12 +38,18 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.distribute.TableDistributedPlanner; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; import org.apache.iotdb.db.queryengine.plan.scheduler.ClusterScheduler; import org.apache.iotdb.db.queryengine.plan.scheduler.IScheduler; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertBaseStatement; +import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; +import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; +import java.util.ArrayList; import java.util.Collections; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; @@ -149,7 +155,39 @@ public ScheduledExecutorService getScheduledExecutorService() { @Override public void setRedirectInfo( - IAnalysis analysis, TEndPoint localEndPoint, TSStatus tsstatus, TSStatusCode statusCode) {} + IAnalysis iAnalysis, TEndPoint localEndPoint, TSStatus tsstatus, TSStatusCode statusCode) { + Analysis analysis = (Analysis) iAnalysis; + if (!(analysis.getStatement() instanceof WrappedInsertStatement)) { + return; + } + InsertBaseStatement insertStatement = + ((WrappedInsertStatement) analysis.getStatement()).getInnerTreeStatement(); + + if (!analysis.isFinishQueryAfterAnalyze()) { + // Table Model Session only supports insertTablet + if (insertStatement instanceof InsertTabletStatement) { + if (statusCode == TSStatusCode.SUCCESS_STATUS) { + boolean needRedirect = false; + List redirectNodeList = analysis.getRedirectNodeList(); + List subStatus = new ArrayList<>(redirectNodeList.size()); + for (TEndPoint endPoint : redirectNodeList) { + // redirect writing only if the redirectEndPoint is not the current node + if (!localEndPoint.equals(endPoint)) { + subStatus.add( + RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS).setRedirectNode(endPoint)); + needRedirect = true; + } else { + subStatus.add(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + } + } + if (needRedirect) { + tsstatus.setCode(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()); + tsstatus.setSubStatus(subStatus); + } + } + } + } + } private static class NopAccessControl implements AccessControl {} } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java index 976d5ad2de96..14c50cb39372 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanner.java @@ -131,6 +131,13 @@ public PlanNode generateDistributedPlanWithOptimize( } } + // Add all Symbol Types in SymbolAllocator into TypeProvider + // TODO Remove redundant logic in LogicalPlan generation or Optimizer + symbolAllocator + .getTypes() + .allTableModelTypes() + .forEach((k, v) -> mppQueryContext.getTypeProvider().putTableModelType(k, v)); + // add exchange node for distributed plan return new AddExchangeNodes(mppQueryContext).addExchangeNodes(distributedPlan, planContext); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java index 12000b9aec2b..a78f8695b070 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelTypeProviderExtractor.java @@ -91,9 +91,14 @@ public Void visitAggregation(AggregationNode node, Void context) { node.getChild().accept(this, context); node.getAggregations() .forEach( - (k, v) -> - beTypeProvider.putTableModelType( - k, v.getResolvedFunction().getSignature().getReturnType())); + (k, v) -> beTypeProvider.putTableModelType(k, feTypeProvider.getTableModelType(k))); + node.getGroupingKeys() + .forEach( + k -> { + if ((!beTypeProvider.isSymbolExist(k))) { + beTypeProvider.putTableModelType(k, feTypeProvider.getTableModelType(k)); + } + }); return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java index 6bc05802df95..771135627c9e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushAggregationIntoTableScan.java @@ -139,7 +139,13 @@ private PushDownLevel calculatePushDownLevel( for (AggregationNode.Aggregation aggregation : values) { // if the function cannot make use of Statistics, we don't push down if (!metadata.canUseStatistics( - aggregation.getResolvedFunction().getSignature().getName())) { + aggregation.getResolvedFunction().getSignature().getName(), + aggregation.getArguments().stream() + .anyMatch( + v -> + ((SymbolReference) v) + .getName() + .equalsIgnoreCase(TimestampOperand.TIMESTAMP_EXPRESSION_STRING)))) { return PushDownLevel.NOOP; } @@ -156,7 +162,11 @@ private PushDownLevel calculatePushDownLevel( // calculate DataSet part if (groupingKeys.isEmpty()) { // GlobalAggregation - return PushDownLevel.COMPLETE; + if (tableScanNode.getDeviceEntries().size() < 2) { + return PushDownLevel.COMPLETE; + } + // We need to two-stage Aggregation to combine Aggregation result of different DeviceEntry + return PushDownLevel.PARTIAL; } List dateBinFunctionsOfTime = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushLimitOffsetIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushLimitOffsetIntoTableScan.java index 2d635afd644f..90c2096070b6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushLimitOffsetIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushLimitOffsetIntoTableScan.java @@ -84,10 +84,8 @@ public PlanNode visitJoin(JoinNode node, Context context) { node.setLeftChild(leftChild); node.setRightChild(rightChild); - // TODO(beyyes) when outer, left, right join is introduced, fix the condition - if (node.getJoinType() == JoinNode.JoinType.INNER) { - context.enablePushDown = false; - } + // TODO(beyyes) optimize for outer, left, right join + context.enablePushDown = false; return node; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java index 241e035a0338..d70ce7e0b8f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/PushPredicateIntoTableScan.java @@ -88,7 +88,6 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils.extractJoinPredicate; import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils.joinEqualityExpression; import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils.processInnerJoin; -import static org.apache.iotdb.db.queryengine.plan.relational.planner.optimizations.JoinUtils.tryNormalizeToOuterToInnerJoin; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; /** @@ -519,7 +518,7 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { context.inheritedPredicate != null ? context.inheritedPredicate : TRUE_LITERAL; // See if we can rewrite outer joins in terms of a plain inner join - node = tryNormalizeToOuterToInnerJoin(node, inheritedPredicate); + // node = tryNormalizeToOuterToInnerJoin(node, inheritedPredicate); Expression leftEffectivePredicate = TRUE_LITERAL; // effectivePredicateExtractor.extract(session, node.getLeftChild(), types, typeAnalyzer); @@ -548,6 +547,12 @@ public PlanNode visitJoin(JoinNode node, RewriteContext context) { postJoinPredicate = innerJoinPushDownResult.getPostJoinPredicate(); newJoinPredicate = innerJoinPushDownResult.getJoinPredicate(); break; + case FULL: + leftPredicate = TRUE_LITERAL; + rightPredicate = TRUE_LITERAL; + postJoinPredicate = inheritedPredicate; + newJoinPredicate = joinPredicate; + break; default: throw new IllegalStateException("Only support INNER JOIN in current version"); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java index 4cac56ec1762..c256e530ef60 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/Util.java @@ -54,14 +54,15 @@ public static Pair split( List intermediateTypes = TableBuiltinAggregationFunction.getIntermediateTypes( resolvedFunction.getSignature().getName(), - resolvedFunction.getSignature().getReturnType()); + resolvedFunction.getSignature().getArgumentTypes()); Type intermediateType = intermediateTypes.size() == 1 ? intermediateTypes.get(0) : RowType.anonymous(intermediateTypes); Symbol intermediateSymbol = symbolAllocator.newSymbol(resolvedFunction.getSignature().getName(), intermediateType); - + // TODO put symbol and its type to TypeProvide or later process: add all map contents of + // SymbolAllocator to the TypeProvider checkState( !originalAggregation.getOrderingScheme().isPresent(), "Aggregate with ORDER BY does not support partial aggregation"); @@ -123,7 +124,7 @@ public static Pair split( List intermediateTypes = TableBuiltinAggregationFunction.getIntermediateTypes( resolvedFunction.getSignature().getName(), - resolvedFunction.getSignature().getReturnType()); + resolvedFunction.getSignature().getArgumentTypes()); Type intermediateType = intermediateTypes.size() == 1 ? intermediateTypes.get(0) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java new file mode 100644 index 000000000000..4d398b55fcc2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import java.util.Map; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class AlterPipe extends PipeStatement { + + private final String pipeName; + private final boolean ifExistsCondition; + private final Map extractorAttributes; + private final Map processorAttributes; + private final Map connectorAttributes; + private final boolean isReplaceAllExtractorAttributes; + private final boolean isReplaceAllProcessorAttributes; + private final boolean isReplaceAllConnectorAttributes; + + public AlterPipe( + final String pipeName, + final boolean ifExistsCondition, + final Map extractorAttributes, + final Map processorAttributes, + final Map connectorAttributes, + final boolean isReplaceAllExtractorAttributes, + final boolean isReplaceAllProcessorAttributes, + final boolean isReplaceAllConnectorAttributes) { + this.pipeName = requireNonNull(pipeName); + this.ifExistsCondition = ifExistsCondition; + this.extractorAttributes = requireNonNull(extractorAttributes); + this.processorAttributes = requireNonNull(processorAttributes); + this.connectorAttributes = requireNonNull(connectorAttributes); + this.isReplaceAllExtractorAttributes = isReplaceAllExtractorAttributes; + this.isReplaceAllProcessorAttributes = isReplaceAllProcessorAttributes; + this.isReplaceAllConnectorAttributes = isReplaceAllConnectorAttributes; + } + + public String getPipeName() { + return pipeName; + } + + public boolean hasIfExistsCondition() { + return ifExistsCondition; + } + + public Map getExtractorAttributes() { + return extractorAttributes; + } + + public Map getProcessorAttributes() { + return processorAttributes; + } + + public Map getConnectorAttributes() { + return connectorAttributes; + } + + public boolean isReplaceAllExtractorAttributes() { + return isReplaceAllExtractorAttributes; + } + + public boolean isReplaceAllProcessorAttributes() { + return isReplaceAllProcessorAttributes; + } + + public boolean isReplaceAllConnectorAttributes() { + return isReplaceAllConnectorAttributes; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitAlterPipe(this, context); + } + + @Override + public int hashCode() { + return Objects.hash( + pipeName, + ifExistsCondition, + extractorAttributes, + processorAttributes, + connectorAttributes, + isReplaceAllExtractorAttributes, + isReplaceAllProcessorAttributes, + isReplaceAllConnectorAttributes); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + AlterPipe alterPipe = (AlterPipe) obj; + return Objects.equals(pipeName, alterPipe.pipeName) + && Objects.equals(ifExistsCondition, alterPipe.ifExistsCondition) + && Objects.equals(extractorAttributes, alterPipe.extractorAttributes) + && Objects.equals(processorAttributes, alterPipe.processorAttributes) + && Objects.equals(connectorAttributes, alterPipe.connectorAttributes) + && Objects.equals( + isReplaceAllExtractorAttributes, alterPipe.isReplaceAllExtractorAttributes) + && Objects.equals( + isReplaceAllProcessorAttributes, alterPipe.isReplaceAllProcessorAttributes) + && Objects.equals( + isReplaceAllConnectorAttributes, alterPipe.isReplaceAllConnectorAttributes); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("pipeName", pipeName) + .add("ifExistsCondition", ifExistsCondition) + .add("extractorAttributes", extractorAttributes) + .add("processorAttributes", processorAttributes) + .add("connectorAttributes", connectorAttributes) + .add("isReplaceAllExtractorAttributes", isReplaceAllExtractorAttributes) + .add("isReplaceAllProcessorAttributes", isReplaceAllProcessorAttributes) + .add("isReplaceAllConnectorAttributes", isReplaceAllConnectorAttributes) + .toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 6ee3137851c2..a2cd774e26b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -347,6 +347,10 @@ protected R visitShowConfigNodes(ShowConfigNodes node, C context) { return visitStatement(node, context); } + protected R visitShowAINodes(ShowAINodes node, C context) { + return visitStatement(node, context); + } + protected R visitRenameTable(RenameTable node, C context) { return visitStatement(node, context); } @@ -498,4 +502,40 @@ protected R visitShowDevice(ShowDevice node, C context) { protected R visitCountDevice(CountDevice node, C context) { return visitStatement(node, context); } + + protected R visitCreatePipe(CreatePipe node, C context) { + return visitStatement(node, context); + } + + protected R visitAlterPipe(AlterPipe node, C context) { + return visitStatement(node, context); + } + + protected R visitDropPipe(DropPipe node, C context) { + return visitStatement(node, context); + } + + protected R visitStartPipe(StartPipe node, C context) { + return visitStatement(node, context); + } + + protected R visitStopPipe(StopPipe node, C context) { + return visitStatement(node, context); + } + + protected R visitShowPipes(ShowPipes node, C context) { + return visitStatement(node, context); + } + + protected R visitCreatePipePlugin(CreatePipePlugin node, C context) { + return visitStatement(node, context); + } + + protected R visitDropPipePlugin(DropPipePlugin node, C context) { + return visitStatement(node, context); + } + + protected R visitShowPipePlugins(ShowPipePlugins node, C context) { + return visitStatement(node, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CoalesceExpression.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CoalesceExpression.java index 4a3365207fff..d16ab376698d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CoalesceExpression.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CoalesceExpression.java @@ -20,9 +20,14 @@ package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.ReadWriteIOUtils; import javax.annotation.Nonnull; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -89,4 +94,27 @@ public int hashCode() { public boolean shallowEquals(Node other) { return sameClass(this, other); } + + // =============== serialize ================= + @Override + public TableExpressionType getExpressionType() { + return TableExpressionType.COALESCE; + } + + @Override + protected void serialize(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(operands.size(), stream); + for (Expression operand : operands) { + serialize(operand, stream); + } + } + + public CoalesceExpression(ByteBuffer byteBuffer) { + super(null); + int size = ReadWriteIOUtils.readInt(byteBuffer); + this.operands = new ArrayList<>(size); + while (size-- > 0) { + operands.add(deserialize(byteBuffer)); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreatePipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreatePipe.java new file mode 100644 index 000000000000..cda5d9c5fd71 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreatePipe.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import java.util.Map; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class CreatePipe extends PipeStatement { + + private final String pipeName; + private final boolean ifNotExistsCondition; + private final Map extractorAttributes; + private final Map processorAttributes; + private final Map connectorAttributes; + + public CreatePipe( + final String pipeName, + final boolean ifNotExistsCondition, + final Map extractorAttributes, + final Map processorAttributes, + final Map connectorAttributes) { + this.pipeName = requireNonNull(pipeName, "pipe name can not be null"); + this.ifNotExistsCondition = ifNotExistsCondition; + this.extractorAttributes = + requireNonNull(extractorAttributes, "extractor/source attributes can not be null"); + this.processorAttributes = + requireNonNull(processorAttributes, "processor attributes can not be null"); + this.connectorAttributes = + requireNonNull(connectorAttributes, "connector attributes can not be null"); + } + + public String getPipeName() { + return pipeName; + } + + public boolean hasIfNotExistsCondition() { + return ifNotExistsCondition; + } + + public Map getExtractorAttributes() { + return extractorAttributes; + } + + public Map getProcessorAttributes() { + return processorAttributes; + } + + public Map getConnectorAttributes() { + return connectorAttributes; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitCreatePipe(this, context); + } + + @Override + public int hashCode() { + return Objects.hash( + pipeName, + ifNotExistsCondition, + extractorAttributes, + processorAttributes, + connectorAttributes); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + CreatePipe other = (CreatePipe) obj; + return Objects.equals(pipeName, other.pipeName) + && Objects.equals(ifNotExistsCondition, other.ifNotExistsCondition) + && Objects.equals(extractorAttributes, other.extractorAttributes) + && Objects.equals(processorAttributes, other.processorAttributes) + && Objects.equals(connectorAttributes, other.connectorAttributes); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("pipeName", pipeName) + .add("ifNotExistsCondition", ifNotExistsCondition) + .add("extractorAttributes", extractorAttributes) + .add("processorAttributes", processorAttributes) + .add("connectorAttributes", connectorAttributes) + .toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreatePipePlugin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreatePipePlugin.java new file mode 100644 index 000000000000..676e360b3ab1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/CreatePipePlugin.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class CreatePipePlugin extends PipeStatement { + + private final String pluginName; + private final boolean ifNotExistsCondition; + private final String className; + private final String uriString; + + public CreatePipePlugin( + final String pluginName, + final boolean ifNotExistsCondition, + final String className, + final String uriString) { + this.pluginName = requireNonNull(pluginName, "plugin name can not be null"); + this.ifNotExistsCondition = ifNotExistsCondition; + this.className = requireNonNull(className, "class name can not be null"); + this.uriString = requireNonNull(uriString, "uri can not be null"); + } + + public String getPluginName() { + return pluginName; + } + + public boolean hasIfNotExistsCondition() { + return ifNotExistsCondition; + } + + public String getClassName() { + return className; + } + + public String getUriString() { + return uriString; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitCreatePipePlugin(this, context); + } + + @Override + public int hashCode() { + return Objects.hash(pluginName, ifNotExistsCondition, className, uriString); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + CreatePipePlugin other = (CreatePipePlugin) obj; + return Objects.equals(pluginName, other.pluginName) + && Objects.equals(ifNotExistsCondition, other.ifNotExistsCondition) + && Objects.equals(className, other.className) + && Objects.equals(uriString, other.uriString); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("pluginName", pluginName) + .add("ifNotExistsCondition", ifNotExistsCondition) + .add("className", className) + .add("uriString", uriString) + .toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java new file mode 100644 index 000000000000..6952c06d13d6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class DropPipe extends PipeStatement { + + private final String pipeName; + private final boolean ifExistsCondition; + + public DropPipe(final String pipeName, final boolean ifExistsCondition) { + this.pipeName = requireNonNull(pipeName, "pipe name can not be null"); + this.ifExistsCondition = ifExistsCondition; + } + + public String getPipeName() { + return pipeName; + } + + public boolean hasIfExistsCondition() { + return ifExistsCondition; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitDropPipe(this, context); + } + + @Override + public int hashCode() { + return Objects.hash(pipeName, ifExistsCondition); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + DropPipe other = (DropPipe) obj; + return Objects.equals(pipeName, other.pipeName) + && Objects.equals(ifExistsCondition, other.ifExistsCondition); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("pipeName", pipeName) + .add("ifExistsCondition", ifExistsCondition) + .toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipePlugin.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipePlugin.java new file mode 100644 index 000000000000..aaa7de52d214 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipePlugin.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class DropPipePlugin extends PipeStatement { + + private final String pluginName; + private final boolean ifExistsCondition; + + public DropPipePlugin(final String pluginName, final boolean ifExistsCondition) { + this.pluginName = requireNonNull(pluginName, "plugin name can not be null"); + this.ifExistsCondition = ifExistsCondition; + } + + public String getPluginName() { + return pluginName; + } + + public boolean hasIfExistsCondition() { + return ifExistsCondition; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitDropPipePlugin(this, context); + } + + @Override + public int hashCode() { + return Objects.hash(pluginName, ifExistsCondition); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + DropPipePlugin other = (DropPipePlugin) obj; + return Objects.equals(pluginName, other.pluginName) + && Objects.equals(ifExistsCondition, other.ifExistsCondition); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("pluginName", pluginName) + .add("ifExistsCondition", ifExistsCondition) + .toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java index 694cf6ca194a..b17a3646b74f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/Expression.java @@ -146,6 +146,9 @@ public static Expression deserialize(ByteBuffer byteBuffer) { case 24: expression = new SymbolReference(byteBuffer); break; + case 25: + expression = new CoalesceExpression(byteBuffer); + break; default: throw new IllegalArgumentException("Invalid expression type: " + type); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/IAINodeRPCServiceWithHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PipeStatement.java similarity index 70% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/IAINodeRPCServiceWithHandler.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PipeStatement.java index 7d9df50ac3ce..2a7c9279f7e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/IAINodeRPCServiceWithHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PipeStatement.java @@ -17,10 +17,20 @@ * under the License. */ -package org.apache.iotdb.db.protocol.thrift.impl; +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; -import org.apache.iotdb.mpp.rpc.thrift.IAINodeInternalRPCService; +import com.google.common.collect.ImmutableList; -public interface IAINodeRPCServiceWithHandler extends IAINodeInternalRPCService.Iface { - void handleExit(); +import java.util.List; + +public abstract class PipeStatement extends Statement { + + protected PipeStatement() { + super(null); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowAINodes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowAINodes.java new file mode 100644 index 000000000000..baaaa5eea5e2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowAINodes.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class ShowAINodes extends Statement { + + public ShowAINodes() { + super(null); + } + + public ShowAINodes(NodeLocation location) { + super(requireNonNull(location, "location is null")); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitShowAINodes(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + return (obj != null) && (getClass() == obj.getClass()); + } + + @Override + public String toString() { + return toStringHelper(this).toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/AINodeRPCServiceMBean.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipePlugins.java similarity index 58% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/AINodeRPCServiceMBean.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipePlugins.java index f4f51c0caa27..115a554a227c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/AINodeRPCServiceMBean.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipePlugins.java @@ -17,6 +17,29 @@ * under the License. */ -package org.apache.iotdb.db.service; +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; -public interface AINodeRPCServiceMBean {} +import static com.google.common.base.MoreObjects.toStringHelper; + +public class ShowPipePlugins extends PipeStatement { + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitShowPipePlugins(this, context); + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean equals(Object obj) { + return obj instanceof ShowPipePlugins; + } + + @Override + public String toString() { + return toStringHelper(this).toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java new file mode 100644 index 000000000000..e65079610931 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import javax.annotation.Nullable; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; + +public class ShowPipes extends PipeStatement { + + private final String pipeName; + private final boolean hasWhereClause; + + public ShowPipes(final @Nullable String pipeName, final boolean hasWhereClause) { + this.pipeName = pipeName; + this.hasWhereClause = hasWhereClause; + } + + public String getPipeName() { + return pipeName; + } + + public boolean hasWhereClause() { + return hasWhereClause; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitShowPipes(this, context); + } + + @Override + public int hashCode() { + return Objects.hash(pipeName, hasWhereClause); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ShowPipes other = (ShowPipes) obj; + return Objects.equals(pipeName, other.pipeName) + && Objects.equals(hasWhereClause, other.hasWhereClause); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("pipeName", pipeName) + .add("hasWhereClause", hasWhereClause) + .toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java new file mode 100644 index 000000000000..540d6fef9ec8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class StartPipe extends PipeStatement { + + private final String pipeName; + + public StartPipe(final String pipeName) { + this.pipeName = requireNonNull(pipeName, "pipe name can not be null"); + } + + public String getPipeName() { + return pipeName; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitStartPipe(this, context); + } + + @Override + public int hashCode() { + return Objects.hash(pipeName); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + StartPipe other = (StartPipe) obj; + return Objects.equals(pipeName, other.pipeName); + } + + @Override + public String toString() { + return toStringHelper(this).add("pipeName", pipeName).toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java new file mode 100644 index 000000000000..daa9ab32e02b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class StopPipe extends PipeStatement { + + private final String pipeName; + + public StopPipe(final String pipeName) { + this.pipeName = requireNonNull(pipeName, "pipe name can not be null"); + } + + public String getPipeName() { + return pipeName; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitStopPipe(this, context); + } + + @Override + public int hashCode() { + return Objects.hash(pipeName); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + StopPipe other = (StopPipe) obj; + return Objects.equals(pipeName, other.pipeName); + } + + @Override + public String toString() { + return toStringHelper(this).add("pipeName", pipeName).toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java index d3700c157b73..56f7470b6519 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/TableExpressionType.java @@ -43,7 +43,8 @@ public enum TableExpressionType { LONG_LITERAL((short) 21), NULL_LITERAL((short) 22), STRING_LITERAL((short) 23), - SYMBOL_REFERENCE((short) 24); + SYMBOL_REFERENCE((short) 24), + COALESCE((short) 25); TableExpressionType(short type) { this.type = type; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 7df892dbeda4..c2262423b4a3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -26,10 +26,12 @@ import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimestampOperand; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AddColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AliasedRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllColumns; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllRows; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticBinaryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticUnaryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BetweenPredicate; @@ -42,6 +44,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CountDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateIndex; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipe; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipePlugin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentDatabase; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentTime; @@ -55,6 +59,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropIndex; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipe; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipePlugin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Except; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; @@ -109,19 +115,24 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowAINodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowConfigNodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDataNodes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDevice; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowIndex; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipePlugins; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowRegions; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTables; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; @@ -159,6 +170,8 @@ import javax.annotation.Nullable; +import java.net.URI; +import java.net.URISyntaxException; import java.time.ZoneId; import java.util.ArrayDeque; import java.util.ArrayList; @@ -168,6 +181,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -547,6 +561,187 @@ public Node visitLoadTsFileStatement(RelationalSqlParser.LoadTsFileStatementCont return super.visitLoadTsFileStatement(ctx); } + @Override + public Node visitCreatePipeStatement(RelationalSqlParser.CreatePipeStatementContext ctx) { + final String pipeName = ((Identifier) visit(ctx.identifier())).getValue(); + final boolean hasIfNotExistsCondition = + ctx.IF() != null && ctx.NOT() != null && ctx.EXISTS() != null; + + final Map extractorAttributes = + ctx.extractorAttributesClause() != null + ? parseExtractorAttributesClause( + ctx.extractorAttributesClause().extractorAttributeClause()) + : Collections.emptyMap(); + final Map processorAttributes = + ctx.processorAttributesClause() != null + ? parseProcessorAttributesClause( + ctx.processorAttributesClause().processorAttributeClause()) + : Collections.emptyMap(); + final Map connectorAttributes = + ctx.connectorAttributesClause() != null + ? parseConnectorAttributesClause( + ctx.connectorAttributesClause().connectorAttributeClause()) + : parseConnectorAttributesClause( + ctx.connectorAttributesWithoutWithSinkClause().connectorAttributeClause()); + + return new CreatePipe( + pipeName, + hasIfNotExistsCondition, + extractorAttributes, + processorAttributes, + connectorAttributes); + } + + private Map parseExtractorAttributesClause( + List contexts) { + final Map collectorMap = new HashMap<>(); + for (RelationalSqlParser.ExtractorAttributeClauseContext context : contexts) { + collectorMap.put( + ((StringLiteral) visit(context.extractorKey)).getValue(), + ((StringLiteral) visit(context.extractorValue)).getValue()); + } + return collectorMap; + } + + private Map parseProcessorAttributesClause( + List contexts) { + final Map processorMap = new HashMap<>(); + for (RelationalSqlParser.ProcessorAttributeClauseContext context : contexts) { + processorMap.put( + ((StringLiteral) visit(context.processorKey)).getValue(), + ((StringLiteral) visit(context.processorValue)).getValue()); + } + return processorMap; + } + + private Map parseConnectorAttributesClause( + List contexts) { + final Map connectorMap = new HashMap<>(); + for (RelationalSqlParser.ConnectorAttributeClauseContext context : contexts) { + connectorMap.put( + ((StringLiteral) visit(context.connectorKey)).getValue(), + ((StringLiteral) visit(context.connectorValue)).getValue()); + } + return connectorMap; + } + + @Override + public Node visitAlterPipeStatement(RelationalSqlParser.AlterPipeStatementContext ctx) { + final String pipeName = ((Identifier) visit(ctx.identifier())).getValue(); + final boolean hasIfExistsCondition = ctx.IF() != null && ctx.EXISTS() != null; + + final Map extractorAttributes; + final boolean isReplaceAllExtractorAttributes; + if (ctx.alterExtractorAttributesClause() != null) { + extractorAttributes = + parseExtractorAttributesClause( + ctx.alterExtractorAttributesClause().extractorAttributeClause()); + isReplaceAllExtractorAttributes = + Objects.nonNull(ctx.alterExtractorAttributesClause().REPLACE()); + } else { + extractorAttributes = Collections.emptyMap(); + isReplaceAllExtractorAttributes = false; + } + + final Map processorAttributes; + final boolean isReplaceAllProcessorAttributes; + if (ctx.alterProcessorAttributesClause() != null) { + processorAttributes = + parseProcessorAttributesClause( + ctx.alterProcessorAttributesClause().processorAttributeClause()); + isReplaceAllProcessorAttributes = + Objects.nonNull(ctx.alterProcessorAttributesClause().REPLACE()); + } else { + processorAttributes = Collections.emptyMap(); + isReplaceAllProcessorAttributes = false; + } + + final Map connectorAttributes; + final boolean isReplaceAllConnectorAttributes; + if (ctx.alterConnectorAttributesClause() != null) { + connectorAttributes = + parseConnectorAttributesClause( + ctx.alterConnectorAttributesClause().connectorAttributeClause()); + isReplaceAllConnectorAttributes = + Objects.nonNull(ctx.alterConnectorAttributesClause().REPLACE()); + } else { + connectorAttributes = Collections.emptyMap(); + isReplaceAllConnectorAttributes = false; + } + + return new AlterPipe( + pipeName, + hasIfExistsCondition, + extractorAttributes, + processorAttributes, + connectorAttributes, + isReplaceAllExtractorAttributes, + isReplaceAllProcessorAttributes, + isReplaceAllConnectorAttributes); + } + + @Override + public Node visitDropPipeStatement(RelationalSqlParser.DropPipeStatementContext ctx) { + final String pipeName = ((Identifier) visit(ctx.identifier())).getValue(); + final boolean hasIfExistsCondition = ctx.IF() != null && ctx.EXISTS() != null; + return new DropPipe(pipeName, hasIfExistsCondition); + } + + @Override + public Node visitStartPipeStatement(RelationalSqlParser.StartPipeStatementContext ctx) { + return new StartPipe(((Identifier) visit(ctx.identifier())).getValue()); + } + + @Override + public Node visitStopPipeStatement(RelationalSqlParser.StopPipeStatementContext ctx) { + return new StopPipe(((Identifier) visit(ctx.identifier())).getValue()); + } + + @Override + public Node visitShowPipesStatement(RelationalSqlParser.ShowPipesStatementContext ctx) { + final String pipeName = + getIdentifierIfPresent(ctx.identifier()).map(Identifier::getValue).orElse(null); + final boolean hasWhereClause = ctx.WHERE() != null; + return new ShowPipes(pipeName, hasWhereClause); + } + + @Override + public Node visitCreatePipePluginStatement( + RelationalSqlParser.CreatePipePluginStatementContext ctx) { + final String pluginName = ((Identifier) visit(ctx.identifier())).getValue(); + final boolean hasIfNotExistsCondition = + ctx.IF() != null && ctx.NOT() != null && ctx.EXISTS() != null; + final String className = ((StringLiteral) visit(ctx.className)).getValue(); + final String uriString = parseAndValidateURI(ctx.uriClause()); + return new CreatePipePlugin(pluginName, hasIfNotExistsCondition, className, uriString); + } + + private String parseAndValidateURI(RelationalSqlParser.UriClauseContext ctx) { + final String uriString = + ctx.uri.identifier() != null + ? ((Identifier) visit(ctx.uri.identifier())).getValue() + : ((StringLiteral) visit(ctx.uri.string())).getValue(); + try { + new URI(uriString); + } catch (URISyntaxException e) { + throw new SemanticException(String.format("Invalid URI: %s", uriString)); + } + return uriString; + } + + @Override + public Node visitDropPipePluginStatement(RelationalSqlParser.DropPipePluginStatementContext ctx) { + final String pluginName = ((Identifier) visit(ctx.identifier())).getValue(); + final boolean hasIfExistsCondition = ctx.IF() != null && ctx.EXISTS() != null; + return new DropPipePlugin(pluginName, hasIfExistsCondition); + } + + @Override + public Node visitShowPipePluginsStatement( + RelationalSqlParser.ShowPipePluginsStatementContext ctx) { + return new ShowPipePlugins(); + } + @Override public Node visitShowDevicesStatement(final RelationalSqlParser.ShowDevicesStatementContext ctx) { return new ShowDevice( @@ -608,6 +803,11 @@ public Node visitShowConfigNodesStatement( return new ShowConfigNodes(); } + @Override + public Node visitShowAINodesStatement(RelationalSqlParser.ShowAINodesStatementContext ctx) { + return new ShowAINodes(); + } + @Override public Node visitShowClusterIdStatement(RelationalSqlParser.ShowClusterIdStatementContext ctx) { return super.visitShowClusterIdStatement(ctx); @@ -1545,6 +1745,24 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { new DereferenceExpression(getLocation(ctx.label), (Identifier) visit(ctx.label))); } + if (name.toString().equalsIgnoreCase("first") || name.toString().equalsIgnoreCase("last")) { + if (arguments.size() == 1) { + arguments.add( + new Identifier( + TimestampOperand.TIMESTAMP_EXPRESSION_STRING.toLowerCase(Locale.ENGLISH))); + } else if (arguments.size() == 2) { + check( + arguments + .get(1) + .toString() + .equalsIgnoreCase(TimestampOperand.TIMESTAMP_EXPRESSION_STRING), + "The second argument of 'first' or 'last' function must be 'time'", + ctx); + } else { + throw parseError("Invalid number of arguments for 'first' or 'last' function", ctx); + } + } + return new FunctionCall(getLocation(ctx), name, distinct, arguments); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java index 3753190b27af..ba51ddcd5520 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java @@ -22,15 +22,20 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AddColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AliasedRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllColumns; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ColumnDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateFunction; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipe; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreatePipePlugin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CreateTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Delete; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropFunction; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipe; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipePlugin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Except; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; @@ -61,8 +66,12 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowFunctions; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipePlugins; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowPipes; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTables; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartPipe; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableSubquery; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Union; @@ -748,6 +757,205 @@ protected Void visitDropFunction(DropFunction node, Integer indent) { return null; } + @Override + protected Void visitCreatePipe(CreatePipe node, Integer context) { + builder.append("CREATE PIPE "); + if (node.hasIfNotExistsCondition()) { + builder.append("IF NOT EXISTS "); + } + builder.append(node.getPipeName()); + builder.append(" \n"); + + if (!node.getExtractorAttributes().isEmpty()) { + builder + .append("WITH SOURCE (") + .append("\n") + .append( + node.getExtractorAttributes().entrySet().stream() + .map( + entry -> + indentString(1) + + "\"" + + entry.getKey() + + "\" = \"" + + entry.getValue() + + "\"") + .collect(joining(", " + "\n"))) + .append(")\n"); + } + + if (!node.getProcessorAttributes().isEmpty()) { + builder + .append("WITH PROCESSOR (") + .append("\n") + .append( + node.getProcessorAttributes().entrySet().stream() + .map( + entry -> + indentString(1) + + "\"" + + entry.getKey() + + "\" = \"" + + entry.getValue() + + "\"") + .collect(joining(", " + "\n"))) + .append(")\n"); + } + + if (!node.getConnectorAttributes().isEmpty()) { + builder + .append("WITH SINK (") + .append("\n") + .append( + node.getConnectorAttributes().entrySet().stream() + .map( + entry -> + indentString(1) + + "\"" + + entry.getKey() + + "\" = \"" + + entry.getValue() + + "\"") + .collect(joining(", " + "\n"))) + .append(")"); + } + + return null; + } + + @Override + protected Void visitAlterPipe(AlterPipe node, Integer context) { + builder.append("ALTER PIPE "); + if (node.hasIfExistsCondition()) { + builder.append("IF EXISTS "); + } + builder.append(node.getPipeName()); + builder.append(" \n"); + + builder + .append(node.isReplaceAllExtractorAttributes() ? "REPLACE" : "MODIFY") + .append(" SOURCE (") + .append("\n") + .append( + node.getExtractorAttributes().entrySet().stream() + .map( + entry -> + indentString(1) + + "\"" + + entry.getKey() + + "\" = \"" + + entry.getValue() + + "\"") + .collect(joining(", " + "\n"))) + .append(")\n"); + + builder + .append(node.isReplaceAllProcessorAttributes() ? "REPLACE" : "MODIFY") + .append(" PROCESSOR (") + .append("\n") + .append( + node.getProcessorAttributes().entrySet().stream() + .map( + entry -> + indentString(1) + + "\"" + + entry.getKey() + + "\" = \"" + + entry.getValue() + + "\"") + .collect(joining(", " + "\n"))) + .append(")\n"); + + builder + .append(node.isReplaceAllConnectorAttributes() ? "REPLACE" : "MODIFY") + .append(" SINK (") + .append("\n") + .append( + node.getConnectorAttributes().entrySet().stream() + .map( + entry -> + indentString(1) + + "\"" + + entry.getKey() + + "\" = \"" + + entry.getValue() + + "\"") + .collect(joining(", " + "\n"))) + .append(")"); + + return null; + } + + @Override + protected Void visitDropPipe(DropPipe node, Integer context) { + builder.append("DROP PIPE "); + if (node.hasIfExistsCondition()) { + builder.append("IF EXISTS "); + } + builder.append(node.getPipeName()); + + return null; + } + + @Override + protected Void visitStartPipe(StartPipe node, Integer context) { + builder.append("START PIPE ").append(node.getPipeName()); + + return null; + } + + @Override + protected Void visitStopPipe(StopPipe node, Integer context) { + builder.append("STOP PIPE ").append(node.getPipeName()); + + return null; + } + + @Override + protected Void visitShowPipes(ShowPipes node, Integer context) { + builder.append("SHOW PIPES"); + + return null; + } + + @Override + protected Void visitCreatePipePlugin(CreatePipePlugin node, Integer context) { + builder.append("CREATE PIPEPLUGIN "); + if (node.hasIfNotExistsCondition()) { + builder.append("IF NOT EXISTS "); + } + builder.append(node.getPluginName()); + builder.append("\n"); + + builder.append("AS \""); + builder.append(node.getClassName()); + builder.append("\"\n"); + + builder.append("USING URI \""); + builder.append(node.getUriString()); + builder.append("\""); + + return null; + } + + @Override + protected Void visitDropPipePlugin(DropPipePlugin node, Integer context) { + builder.append("DROP PIPEPLUGIN "); + if (node.hasIfExistsCondition()) { + builder.append("IF EXISTS "); + } + builder.append(node.getPluginName()); + + return null; + } + + @Override + protected Void visitShowPipePlugins(ShowPipePlugins node, Integer context) { + builder.append("SHOW PIPEPLUGINS"); + + return null; + } + private void appendBeginLabel(Optional label) { label.ifPresent(value -> builder.append(formatName(value)).append(": ")); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java index 72f8594aa60c..80c25e3df2d2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/type/InternalTypeManager.java @@ -99,6 +99,7 @@ public static TSDataType getTSDataType(Type type) { case TIMESTAMP: return TSDataType.TIMESTAMP; case BLOB: + case ROW: return TSDataType.BLOB; case STRING: return TSDataType.STRING; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/CoalesceColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/CoalesceColumnTransformer.java new file mode 100644 index 000000000000..f88c60ba74dd --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/multi/CoalesceColumnTransformer.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.transformation.dag.column.multi; + +import org.apache.iotdb.db.queryengine.transformation.dag.column.ColumnTransformer; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.block.column.ColumnBuilder; +import org.apache.tsfile.read.common.type.Type; + +import java.util.List; + +public class CoalesceColumnTransformer extends MultiColumnTransformer { + + public CoalesceColumnTransformer(Type returnType, List columnTransformerList) { + super(returnType, columnTransformerList); + } + + @Override + protected void doTransform( + List childrenColumns, ColumnBuilder builder, int positionCount) { + for (int i = 0; i < positionCount; i++) { + boolean allNull = true; + for (Column column : childrenColumns) { + if (!column.isNull(i)) { + allNull = false; + builder.write(column, i); + break; + } + } + if (allNull) { + builder.appendNull(); + } + } + } + + @Override + protected void checkType() { + // do nothing, has checked in FE + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/MetaFormatUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/MetaFormatUtils.java index 1b7017e69405..579fb8f9cb65 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/MetaFormatUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/utils/MetaFormatUtils.java @@ -102,11 +102,12 @@ public static void checkSchemaMeasurementNames(List measurements) } /** check whether the database name uses illegal characters */ - public static void checkDatabase(String database) throws IllegalPathException { + public static void checkDatabase(final String database) throws IllegalPathException { if (!IoTDBConfig.STORAGE_GROUP_PATTERN.matcher(database).matches()) { throw new IllegalPathException( String.format( - "The database name can only be characters, numbers and underscores. %s", database)); + "The database name can only contain english or chinese characters, numbers, backticks and underscores. %s", + database)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index 2e945590dc11..8f8e7c5f14c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -23,16 +23,23 @@ import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.confignode.rpc.thrift.TFetchTableResp; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.ClusterConfigTaskExecutor; +import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; +import javax.annotation.Nonnull; + +import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Optional; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; import java.util.stream.Collectors; @@ -47,9 +54,13 @@ public class DataNodeTableCache implements ITableCache { private final Map> databaseTableMap = new ConcurrentHashMap<>(); - private final Map> preUpdateTableMap = new ConcurrentHashMap<>(); + private final Map>> preUpdateTableMap = + new ConcurrentHashMap<>(); private final ReentrantReadWriteLock readWriteLock = new ReentrantReadWriteLock(); + private final Semaphore fetchTableSemaphore = + new Semaphore( + IoTDBDescriptor.getInstance().getConfig().getDataNodeTableCacheSemaphorePermitNum()); private DataNodeTableCache() { // Do nothing @@ -76,38 +87,52 @@ public void init(final byte[] tableInitializationBytes) { TsTableInternalRPCUtil.deserializeTableInitializationInfo(tableInitializationBytes); final Map> usingMap = tableInfo.left; final Map> preCreateMap = tableInfo.right; - saveUpdatedTableInfo(usingMap, databaseTableMap); - saveUpdatedTableInfo(preCreateMap, preUpdateTableMap); + usingMap.forEach( + (key, value) -> + databaseTableMap.put( + PathUtils.unQualifyDatabaseName(key), + value.stream() + .collect( + Collectors.toMap( + TsTable::getTableName, + Function.identity(), + (v1, v2) -> v2, + ConcurrentHashMap::new)))); + preCreateMap.forEach( + (key, value) -> + preUpdateTableMap.put( + PathUtils.unQualifyDatabaseName(key), + value.stream() + .collect( + Collectors.toMap( + TsTable::getTableName, + table -> new Pair<>(table, 0L), + (v1, v2) -> v2, + ConcurrentHashMap::new)))); LOGGER.info("Init DataNodeTableCache successfully"); } finally { readWriteLock.writeLock().unlock(); } } - private void saveUpdatedTableInfo( - final Map> tableMap, - final Map> localTableMap) { - tableMap.forEach( - (key, value) -> - localTableMap.put( - key, - value.stream() - .collect( - Collectors.toMap( - TsTable::getTableName, - Function.identity(), - (v1, v2) -> v2, - ConcurrentHashMap::new)))); - } - @Override public void preUpdateTable(String database, final TsTable table) { - database = PathUtils.qualifyDatabaseName(database); + database = PathUtils.unQualifyDatabaseName(database); readWriteLock.writeLock().lock(); try { preUpdateTableMap .computeIfAbsent(database, k -> new ConcurrentHashMap<>()) - .put(table.getTableName(), table); + .compute( + table.getTableName(), + (k, v) -> { + if (Objects.isNull(v)) { + return new Pair<>(table, 0L); + } else { + v.setLeft(table); + v.setRight(v.getRight() + 1); + return v; + } + }); LOGGER.info("Pre-update table {}.{} successfully", database, table); } finally { readWriteLock.writeLock().unlock(); @@ -116,7 +141,7 @@ public void preUpdateTable(String database, final TsTable table) { @Override public void rollbackUpdateTable(String database, final String tableName) { - database = PathUtils.qualifyDatabaseName(database); + database = PathUtils.unQualifyDatabaseName(database); readWriteLock.writeLock().lock(); try { removeTableFromPreUpdateMap(database, tableName); @@ -133,24 +158,19 @@ private void removeTableFromPreUpdateMap(final String database, final String tab if (v == null) { throw new IllegalStateException(); } - v.remove(tableName); - if (v.isEmpty()) { - return null; - } else { - return v; - } + v.get(tableName).setLeft(null); + return v; }); } @Override public void commitUpdateTable(String database, final String tableName) { - database = PathUtils.qualifyDatabaseName(database); + database = PathUtils.unQualifyDatabaseName(database); readWriteLock.writeLock().lock(); try { - final TsTable table = preUpdateTableMap.get(database).get(tableName); databaseTableMap .computeIfAbsent(database, k -> new ConcurrentHashMap<>()) - .put(tableName, table); + .put(tableName, preUpdateTableMap.get(database).get(tableName).getLeft()); removeTableFromPreUpdateMap(database, tableName); LOGGER.info("Commit-update table {}.{} successfully", database, tableName); } finally { @@ -160,7 +180,7 @@ public void commitUpdateTable(String database, final String tableName) { @Override public void invalid(String database) { - database = PathUtils.qualifyDatabaseName(database); + database = PathUtils.unQualifyDatabaseName(database); readWriteLock.writeLock().lock(); try { databaseTableMap.remove(database); @@ -170,30 +190,137 @@ public void invalid(String database) { } } + public TsTable getTableInWrite(final String database, final String tableName) { + final TsTable result = getTableInCache(database, tableName); + return Objects.nonNull(result) ? result : getTable(database, tableName); + } + + /** + * The following logic can handle the cases when configNode failed to clear some table in {@link + * #preUpdateTableMap}, due to the failure of "commit" or rollback of "pre-update". + */ public TsTable getTable(String database, final String tableName) { - database = PathUtils.qualifyDatabaseName(database); + database = PathUtils.unQualifyDatabaseName(database); + final Map> preUpdateTables = + mayGetTableInPreUpdateMap(database, tableName); + if (Objects.nonNull(preUpdateTables)) { + updateTable(getTablesInConfigNode(preUpdateTables), preUpdateTables); + } + return getTableInCache(database, tableName); + } + + private Map> mayGetTableInPreUpdateMap( + final String database, final String tableName) { readWriteLock.readLock().lock(); try { - if (databaseTableMap.containsKey(database)) { - return databaseTableMap.get(database).get(tableName); - } - return null; + return preUpdateTableMap.containsKey(database) + && preUpdateTableMap.get(database).containsKey(tableName) + ? preUpdateTableMap.entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + entry -> + entry.getValue().entrySet().stream() + .collect( + Collectors.toMap( + Map.Entry::getKey, + innerEntry -> innerEntry.getValue().getRight())))) + : null; } finally { readWriteLock.readLock().unlock(); } } - public Optional> getTables(String database) { - database = PathUtils.qualifyDatabaseName(database); + private Map> getTablesInConfigNode( + final Map> tableInput) { + Map> result = Collections.emptyMap(); + try { + fetchTableSemaphore.acquire(); + final TFetchTableResp resp = + ClusterConfigTaskExecutor.getInstance() + .fetchTables( + tableInput.entrySet().stream() + .collect( + Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().keySet()))); + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == resp.getStatus().getCode()) { + result = TsTableInternalRPCUtil.deserializeTsTableFetchResult(resp.getTableInfoMap()); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + LOGGER.warn( + "Interrupted when trying to acquire semaphore when trying to get tables from configNode, ignore."); + } catch (final Exception e) { + fetchTableSemaphore.release(); + throw e; + } + fetchTableSemaphore.release(); + return result; + } + + private void updateTable( + final Map> fetchedTables, + final Map> previousVersions) { + readWriteLock.writeLock().lock(); + try { + fetchedTables.forEach( + (database, tableInfoMap) -> { + if (preUpdateTableMap.containsKey(database)) { + tableInfoMap.forEach( + (tableName, tsTable) -> { + final Pair existingPair = + preUpdateTableMap.get(database).get(tableName); + if (Objects.isNull(existingPair) + || !Objects.equals( + existingPair.getRight(), + previousVersions.get(database).get(tableName))) { + return; + } + existingPair.setLeft(null); + if (Objects.nonNull(tsTable)) { + databaseTableMap + .computeIfAbsent(database, k -> new ConcurrentHashMap<>()) + .put(tableName, tsTable); + } else if (databaseTableMap.containsKey(database)) { + databaseTableMap.get(database).remove(tableName); + } + }); + } + }); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + private TsTable getTableInCache(final String database, final String tableName) { readWriteLock.readLock().lock(); try { - final Map tableMap = databaseTableMap.get(database); - return tableMap != null ? Optional.of(new ArrayList<>(tableMap.values())) : Optional.empty(); + return databaseTableMap.containsKey(database) + ? databaseTableMap.get(database).get(tableName) + : null; } finally { readWriteLock.readLock().unlock(); } } + // Database shall not start with "root" + public String tryGetInternColumnName( + final @Nonnull String database, + final @Nonnull String tableName, + final @Nonnull String columnName) { + if (columnName.isEmpty()) { + return columnName; + } + try { + return databaseTableMap + .get(database) + .get(tableName) + .getColumnSchema(columnName) + .getColumnName(); + } catch (final Exception e) { + return columnName; + } + } + /** Check whether the given path overlap with some table existence. */ public Pair checkTableCreateAndPreCreateOnGivenPath(final PartialPath path) { readWriteLock.writeLock().lock(); @@ -210,9 +337,9 @@ public Pair checkTableCreateAndPreCreateOnGivenPath(final Partia } private Pair checkTableExistenceOnGivenPath( - final String path, final Map> tableMap) { + final String path, final Map> tableMap) { final int dbStartIndex = PATH_ROOT.length() + 1; - for (final Map.Entry> dbEntry : tableMap.entrySet()) { + for (final Map.Entry> dbEntry : tableMap.entrySet()) { final String database = dbEntry.getKey(); if (!(path.startsWith(database, dbStartIndex) && path.charAt(dbStartIndex + database.length()) == PATH_SEPARATOR)) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/AINodeRPCService.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/AINodeRPCService.java deleted file mode 100644 index 5ec49e756602..000000000000 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/AINodeRPCService.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.db.service; - -import org.apache.iotdb.commons.concurrent.ThreadName; -import org.apache.iotdb.commons.exception.runtime.RPCServiceException; -import org.apache.iotdb.commons.service.ServiceType; -import org.apache.iotdb.commons.service.ThriftService; -import org.apache.iotdb.commons.service.ThriftServiceThread; -import org.apache.iotdb.db.conf.IoTDBConfig; -import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.protocol.thrift.handler.AINodeRPCServiceThriftHandler; -import org.apache.iotdb.db.protocol.thrift.impl.AINodeRPCServiceImpl; -import org.apache.iotdb.mpp.rpc.thrift.IAINodeInternalRPCService; -import org.apache.iotdb.rpc.DeepCopyRpcTransportFactory; - -public class AINodeRPCService extends ThriftService implements AINodeRPCServiceMBean { - - private AINodeRPCServiceImpl impl; - - private AINodeRPCService() {} - - @Override - public ServiceType getID() { - return ServiceType.AINode_RPC_SERVICE; - } - - @Override - public void initTProcessor() { - impl = new AINodeRPCServiceImpl(); - initSyncedServiceImpl(null); - processor = new IAINodeInternalRPCService.Processor<>(impl); - } - - @Override - public void initThriftServiceThread() - throws IllegalAccessException, InstantiationException, ClassNotFoundException { - try { - IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - thriftServiceThread = - new ThriftServiceThread( - processor, - getID().getName(), - ThreadName.AINODE_RPC_SERVICE.getName(), - getBindIP(), - getBindPort(), - config.getRpcMaxConcurrentClientNum(), - config.getThriftServerAwaitTimeForStopService(), - new AINodeRPCServiceThriftHandler(impl), - config.isRpcThriftCompressionEnable(), - DeepCopyRpcTransportFactory.INSTANCE); - } catch (RPCServiceException e) { - throw new IllegalAccessException(e.getMessage()); - } - thriftServiceThread.setName(ThreadName.AINODE_RPC_SERVICE.getName()); - } - - @Override - public String getBindIP() { - return IoTDBDescriptor.getInstance().getConfig().getRpcAddress(); - } - - @Override - public int getBindPort() { - return IoTDBDescriptor.getInstance().getConfig().getAINodePort(); - } - - private static class AINodeRPCServiceHolder { - private static final AINodeRPCService INSTANCE = new AINodeRPCService(); - - private AINodeRPCServiceHolder() {} - } - - public static AINodeRPCService getInstance() { - return AINodeRPCServiceHolder.INSTANCE; - } -} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java index 1fea2a7e3587..e2d4ff3ce453 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java @@ -819,10 +819,6 @@ private void setUp() throws StartupException { private void setUpRPCService() throws StartupException { // Start InternalRPCService to indicate that the current DataNode can accept cluster scheduling registerManager.register(DataNodeInternalRPCService.getInstance()); - // Start InternalRPCService to indicate that the current DataNode can accept request from AINode - if (config.isEnableAINodeService()) { - registerManager.register(AINodeRPCService.getInstance()); - } // Notice: During the period between starting the internal RPC service // and starting the client RPC service , some requests may fail because diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index 2e607620fcdf..9c1ef5c4f09b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -281,6 +281,7 @@ public Map> getLocalDataRegionInfo() { @Override public void start() throws StartupException { + recoverDataRegionNum = 0; // build time Interval to divide time partition initTimePartition(); // create systemDir diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 697cabbc5dc5..c6b55484b8f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -779,7 +779,11 @@ private void continueFailedRenames(File fileFolder, String suffix) throws IOExce private void checkTsFileTime(File tsFile, long currentTime) throws DataRegionException { String[] items = tsFile.getName().replace(TSFILE_SUFFIX, "").split(FILE_NAME_SEPARATOR); long fileTime = Long.parseLong(items[0]); - if (fileTime > currentTime) { + // skip files generated by repair compaction task + long version = Long.parseLong(items[1]); + if (version > 0 + && fileTime > currentTime + && fileTime < RepairUnsortedFileCompactionTask.getInitialAllocatedFileTimestamp()) { throw new DataRegionException( String.format( "data region %s[%s] is down, because the time of tsfile %s is larger than system current time, " @@ -880,7 +884,6 @@ private void recoverSealedTsFiles( new SealedTsFileRecoverPerformer(sealedTsFile)) { recoverPerformer.recover(); sealedTsFile.close(); - tsFileManager.add(sealedTsFile, isSeq); tsFileResourceManager.registerSealedTsFileResource(sealedTsFile); } catch (Throwable e) { logger.error("Fail to recover sealed TsFile {}, skip it.", sealedTsFile.getTsFilePath(), e); @@ -900,10 +903,10 @@ private Callable recoverFilesInPartition( List resourceListForSyncRecover = new ArrayList<>(); Callable asyncRecoverTask = null; for (TsFileResource tsFileResource : resourceList) { + tsFileManager.add(tsFileResource, isSeq); if (fileTimeIndexMap.containsKey(tsFileResource.getTsFileID())) { tsFileResource.setTimeIndex(fileTimeIndexMap.get(tsFileResource.getTsFileID())); tsFileResource.setStatus(TsFileResourceStatus.NORMAL); - tsFileManager.add(tsFileResource, isSeq); resourceListForAsyncRecover.add(tsFileResource); } else { resourceListForSyncRecover.add(tsFileResource); @@ -1759,9 +1762,12 @@ public Future asyncCloseOneTsFileProcessor(boolean sequence, TsFileProcessor || tsFileProcessor.alreadyMarkedClosing()) { return CompletableFuture.completedFuture(null); } + TsFileResource resource = tsFileProcessor.getTsFileResource(); logger.info( - "Async close tsfile: {}", - tsFileProcessor.getTsFileResource().getTsFile().getAbsolutePath()); + "Async close tsfile: {}, file start time: {}, file end time: {}", + resource.getTsFile().getAbsolutePath(), + resource.getFileStartTime(), + resource.getFileEndTime()); Future future; if (sequence) { closingSequenceTsFileProcessor.add(tsFileProcessor); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java index a0f57c5efa44..880ce4ab3575 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/RepairUnsortedFileCompactionPerformer.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.AbstractCompactionWriter; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.RepairUnsortedFileCompactionWriter; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; @@ -34,11 +35,8 @@ /** Used for fixing files which contains internal unsorted data */ public class RepairUnsortedFileCompactionPerformer extends ReadPointCompactionPerformer { - private final boolean rewriteFile; - - public RepairUnsortedFileCompactionPerformer(boolean rewriteFile, boolean ignoreAllNullRows) { + public RepairUnsortedFileCompactionPerformer(boolean ignoreAllNullRows) { super(); - this.rewriteFile = rewriteFile; this.ignoreAllNullRows = ignoreAllNullRows; } @@ -53,7 +51,8 @@ protected AbstractCompactionWriter getCompactionWriter( @Override public void perform() throws Exception { - if (rewriteFile) { + TsFileResource resource = !seqFiles.isEmpty() ? seqFiles.get(0) : unseqFiles.get(0); + if (resource.getTsFileRepairStatus() == TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE) { super.perform(); } else { prepareTargetFile(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java index 75556001fd62..48706fcb4c83 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java @@ -158,7 +158,7 @@ protected void handleException(Logger logger, Exception e) { // these exceptions generally caused by unsorted data, mark all source files as NEED_TO_REPAIR for (TsFileResource resource : unsortedTsFileResources) { if (resource.getTsFileRepairStatus() != TsFileRepairStatus.CAN_NOT_REPAIR) { - resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR); + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_CHECK); } } } else if (e instanceof InterruptedException @@ -474,7 +474,7 @@ protected void validateCompactionResult( : timeDiff; }); List overlapFilesInTimePartition = - RepairDataFileScanUtil.checkTimePartitionHasOverlap(timePartitionSeqFiles); + RepairDataFileScanUtil.checkTimePartitionHasOverlap(timePartitionSeqFiles, true); if (!overlapFilesInTimePartition.isEmpty()) { LOGGER.error( "Failed to pass compaction validation, source seq files: {}, source unseq files: {}, target files: {}", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java index b80f66737a01..7dcd312b1bc6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java @@ -47,6 +47,7 @@ import java.io.IOException; import java.nio.file.Files; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -301,6 +302,8 @@ private boolean shouldRollback() { private void rollback() throws IOException { // if the task has started, + targetTsfileResourceList = + targetTsfileResourceList == null ? Collections.emptyList() : targetTsfileResourceList; if (recoverMemoryStatus) { replaceTsFileInMemory( targetTsfileResourceList, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java index e2f31a28ffa3..1f7e2b9c0183 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java @@ -121,11 +121,13 @@ protected boolean doCompaction() { recoverMemoryStatus = true; LOGGER.info( "{}-{} [Compaction] InsertionCrossSpaceCompaction task starts with unseq file {}, " + + "nearest seq files are {}, " + "target file name timestamp is {}, " + "file size is {} MB.", storageGroupName, dataRegionId, unseqFileToInsert, + selectedSeqFiles, timestamp, unseqFileToInsert.getTsFileSize() / 1024 / 1024); boolean isSuccess = true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java index e2ac24b0dbae..fe6ae62fd213 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/RepairUnsortedFileCompactionTask.java @@ -24,6 +24,7 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.RepairUnsortedFileCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger; +import org.apache.iotdb.db.storageengine.dataregion.compaction.repair.RepairDataFileScanUtil; import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.estimator.RepairUnsortedFileCompactionEstimator; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus; @@ -54,8 +55,11 @@ public static void recoverAllocatedFileTimestamp(long timestamp) { } } + public static long getInitialAllocatedFileTimestamp() { + return Long.MAX_VALUE / 2; + } + private final TsFileResource sourceFile; - private final boolean rewriteFile; private CountDownLatch latch; public RepairUnsortedFileCompactionTask( @@ -70,63 +74,21 @@ public RepairUnsortedFileCompactionTask( tsFileManager, Collections.singletonList(sourceFile), sequence, - new RepairUnsortedFileCompactionPerformer(true, ignoreAllNullRows), + new RepairUnsortedFileCompactionPerformer(ignoreAllNullRows), serialId); this.sourceFile = sourceFile; - this.innerSpaceEstimator = new RepairUnsortedFileCompactionEstimator(); - this.rewriteFile = false; - } - - public RepairUnsortedFileCompactionTask( - long timePartition, - TsFileManager tsFileManager, - TsFileResource sourceFile, - boolean sequence, - boolean rewriteFile, - long serialId, - boolean ignoreAllNullRows) { - super( - timePartition, - tsFileManager, - Collections.singletonList(sourceFile), - sequence, - new RepairUnsortedFileCompactionPerformer(rewriteFile, ignoreAllNullRows), - serialId); - this.sourceFile = sourceFile; - if (rewriteFile) { + if (this.sourceFile.getTsFileRepairStatus() != TsFileRepairStatus.NEED_TO_REPAIR_BY_MOVE) { this.innerSpaceEstimator = new RepairUnsortedFileCompactionEstimator(); } - this.rewriteFile = rewriteFile; - } - - public RepairUnsortedFileCompactionTask( - long timePartition, - TsFileManager tsFileManager, - TsFileResource sourceFile, - CountDownLatch latch, - boolean sequence, - long serialId, - boolean ignoreAllNullRows) { - super( - timePartition, - tsFileManager, - Collections.singletonList(sourceFile), - sequence, - new RepairUnsortedFileCompactionPerformer(true, ignoreAllNullRows), - serialId); - this.sourceFile = sourceFile; - this.innerSpaceEstimator = new RepairUnsortedFileCompactionEstimator(); - this.latch = latch; - this.rewriteFile = false; } + // used for 'start repair data' public RepairUnsortedFileCompactionTask( long timePartition, TsFileManager tsFileManager, TsFileResource sourceFile, CountDownLatch latch, boolean sequence, - boolean rewriteFile, long serialId, boolean ignoreAllNullRows) { super( @@ -134,13 +96,12 @@ public RepairUnsortedFileCompactionTask( tsFileManager, Collections.singletonList(sourceFile), sequence, - new RepairUnsortedFileCompactionPerformer(rewriteFile, ignoreAllNullRows), + new RepairUnsortedFileCompactionPerformer(ignoreAllNullRows), serialId); this.sourceFile = sourceFile; - if (rewriteFile) { + if (this.sourceFile.getTsFileRepairStatus() != TsFileRepairStatus.NEED_TO_REPAIR_BY_MOVE) { this.innerSpaceEstimator = new RepairUnsortedFileCompactionEstimator(); } - this.rewriteFile = rewriteFile; this.latch = latch; } @@ -205,7 +166,7 @@ protected void prepareTargetFiles() throws IOException { storageGroupName, dataRegionId); - if (rewriteFile) { + if (sourceFile.getTsFileRepairStatus() == TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE) { CompactionUtils.combineModsInInnerCompaction( filesView.sourceFilesInCompactionPerformer, filesView.targetFilesInPerformer); } else { @@ -219,6 +180,10 @@ protected void prepareTargetFiles() throws IOException { @Override protected boolean doCompaction() { + calculateRepairMethod(); + if (!sourceFile.getTsFileRepairStatus().isRepairCompactionCandidate()) { + return true; + } boolean isSuccess = super.doCompaction(); if (!isSuccess) { LOGGER.info("Failed to repair file {}", sourceFile.getTsFile().getAbsolutePath()); @@ -227,6 +192,27 @@ protected boolean doCompaction() { return isSuccess; } + private void calculateRepairMethod() { + if (this.sourceFile.getTsFileRepairStatus() != TsFileRepairStatus.NEED_TO_CHECK) { + return; + } + RepairDataFileScanUtil repairDataFileScanUtil = new RepairDataFileScanUtil(sourceFile, true); + repairDataFileScanUtil.scanTsFile(); + if (repairDataFileScanUtil.isBrokenFile()) { + sourceFile.setTsFileRepairStatus(TsFileRepairStatus.CAN_NOT_REPAIR); + return; + } + if (repairDataFileScanUtil.hasUnsortedData()) { + sourceFile.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); + return; + } + if (sourceFile.isSeq()) { + sourceFile.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_MOVE); + return; + } + sourceFile.setTsFileRepairStatus(TsFileRepairStatus.NORMAL); + } + @Override public long getEstimatedMemoryCost() { if (innerSpaceEstimator != null && memoryCost == 0L) { @@ -252,7 +238,7 @@ public long getEstimatedMemoryCost() { @Override public boolean isDiskSpaceCheckPassed() { - if (!rewriteFile) { + if (sourceFile.getTsFileRepairStatus() == TsFileRepairStatus.NEED_TO_REPAIR_BY_MOVE) { return true; } return super.isDiskSpaceCheckPassed(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java index 2a6d6e6099b1..c3641b415b9b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/SettleCompactionTask.java @@ -156,17 +156,25 @@ protected boolean doCompaction() { double costTime = (System.currentTimeMillis() - startTime) / 1000.0d; if (isSuccess) { - LOGGER.info( - "{}-{} [Compaction] SettleCompaction task finishes successfully, time cost is {} s, compaction speed is {} MB/s." - + "Fully_dirty files num is {} and partially_dirty files num is {}.", - storageGroupName, - dataRegionId, - String.format("%.2f", costTime), - String.format( - "%.2f", - (fullyDirtyFileSize + partiallyDirtyFileSize) / 1024.0d / 1024.0d / costTime), - fullyDirtyFiles.size(), - filesView.sourceFilesInCompactionPerformer.size()); + if (partiallyDirtyFileSize == 0) { + LOGGER.info( + "{}-{} [Compaction] SettleCompaction task finishes successfully, time cost is {} s." + + "Fully_dirty files num is {}.", + storageGroupName, + dataRegionId, + String.format("%.2f", costTime), + fullyDirtyFiles.size()); + } else { + LOGGER.info( + "{}-{} [Compaction] SettleCompaction task finishes successfully, time cost is {} s, compaction speed is {} MB/s." + + "Fully_dirty files num is {} and partially_dirty files num is {}.", + storageGroupName, + dataRegionId, + String.format("%.2f", costTime), + String.format("%.2f", (partiallyDirtyFileSize) / 1024.0d / 1024.0d / costTime), + fullyDirtyFiles.size(), + filesView.sourceFilesInCompactionPerformer.size()); + } } else { LOGGER.info( "{}-{} [Compaction] SettleCompaction task finishes with some error, time cost is {} s." diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java index 3e5f69b88e5e..a17e296c274a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/batch/utils/AlignedSeriesBatchCompactionUtils.java @@ -80,13 +80,17 @@ public static AlignedChunkMetadata fillAlignedChunkMetadataBySchemaList( AlignedChunkMetadata originAlignedChunkMetadata, List schemaList) { List originValueChunkMetadataList = originAlignedChunkMetadata.getValueChunkMetadataList(); - if (originValueChunkMetadataList.size() == schemaList.size()) { - return originAlignedChunkMetadata; - } IChunkMetadata[] newValueChunkMetadataArr = new IChunkMetadata[schemaList.size()]; int currentValueChunkMetadataIndex = 0; for (int i = 0; i < schemaList.size(); i++) { IMeasurementSchema currentSchema = schemaList.get(i); + + // skip null value + while (currentValueChunkMetadataIndex < originValueChunkMetadataList.size() + && originValueChunkMetadataList.get(currentValueChunkMetadataIndex) == null) { + currentValueChunkMetadataIndex++; + } + if (currentValueChunkMetadataIndex >= originValueChunkMetadataList.size()) { break; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java index 64439887ae8a..b2a8abb28b0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairDataFileScanUtil.java @@ -69,11 +69,17 @@ public class RepairDataFileScanUtil { private boolean hasUnsortedData; private boolean isBrokenFile; private long previousTime; + private boolean printLog; public RepairDataFileScanUtil(TsFileResource resource) { + this(resource, false); + } + + public RepairDataFileScanUtil(TsFileResource resource, boolean printLog) { this.resource = resource; this.hasUnsortedData = false; this.previousTime = Long.MIN_VALUE; + this.printLog = printLog; } public void scanTsFile() { @@ -99,6 +105,12 @@ public void scanTsFile() { } } catch (CompactionLastTimeCheckFailedException lastTimeCheckFailedException) { this.hasUnsortedData = true; + if (printLog) { + logger.error( + "File {} has unsorted data: ", + resource.getTsFile().getPath(), + lastTimeCheckFailedException); + } } catch (Exception e) { // ignored the exception caused by thread interrupt if (Thread.currentThread().isInterrupted()) { @@ -242,7 +254,8 @@ public boolean isBrokenFile() { return isBrokenFile; } - public static List checkTimePartitionHasOverlap(List resources) { + public static List checkTimePartitionHasOverlap( + List resources, boolean printOverlappedDevices) { List overlapResources = new ArrayList<>(); Map deviceEndTimeMap = new HashMap<>(); for (TsFileResource resource : resources) { @@ -270,6 +283,13 @@ public static List checkTimePartitionHasOverlap(List seqList = tsFileManager.getTsFileListSnapshot(timePartition.getTimePartitionId(), true); List overlapFiles = - RepairDataFileScanUtil.checkTimePartitionHasOverlap(seqList); + RepairDataFileScanUtil.checkTimePartitionHasOverlap(seqList, false); for (TsFileResource overlapFile : overlapFiles) { if (!timePartition.getTsFileManager().isAllowCompaction()) { LOGGER.info( @@ -153,6 +154,7 @@ private void checkOverlapInSequenceSpaceAndRepair(RepairTimePartition timePartit } checkTaskStatusAndMayStop(); CountDownLatch latch = new CountDownLatch(1); + overlapFile.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_MOVE); RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask( timePartition.getTimePartitionId(), @@ -160,7 +162,6 @@ private void checkOverlapInSequenceSpaceAndRepair(RepairTimePartition timePartit overlapFile, latch, true, - false, tsFileManager.getNextCompactionTaskId(), true); LOGGER.info( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/UnsortedFileRepairTaskScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/UnsortedFileRepairTaskScheduler.java index 5cdf9493581e..5993f8bc9b37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/UnsortedFileRepairTaskScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/UnsortedFileRepairTaskScheduler.java @@ -148,7 +148,7 @@ private void recoverRepairProgress(RepairTaskRecoverLogParser recoverLogParser) continue; } if (cannotRepairFiles.contains(resource.getTsFile().getName())) { - resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR); + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java index 4c72c0dd0537..92cccbfb24cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.exception.MergeException; import org.apache.iotdb.db.service.metrics.CompactionMetrics; import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleContext; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.ICompactionSelector; @@ -327,6 +328,9 @@ private boolean canSubmitCrossTask( @Override public List selectCrossSpaceTask( List sequenceFileList, List unsequenceFilelist) { + if (!CompactionUtils.isDiskHasSpace()) { + return Collections.emptyList(); + } return selectCrossSpaceTask(sequenceFileList, unsequenceFilelist, false); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java index a23c4dbf4382..9c5eeebdaf9d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/SizeTieredCompactionSelector.java @@ -26,13 +26,13 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.ICompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.RepairUnsortedFileCompactionTask; +import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionScheduleContext; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.CompactionTaskManager; import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.comparator.ICompactionTaskComparator; import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.IInnerSeqSpaceSelector; import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.IInnerUnseqSpaceSelector; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileRepairStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.generator.TsFileNameGenerator; @@ -167,8 +167,7 @@ private List> selectTsFileResourcesByLevel(int level) throw private boolean cannotSelectCurrentFileToNormalCompaction(TsFileResource resource) { return resource.getStatus() != TsFileResourceStatus.NORMAL - || resource.getTsFileRepairStatus() == TsFileRepairStatus.NEED_TO_REPAIR - || resource.getTsFileRepairStatus() == TsFileRepairStatus.CAN_NOT_REPAIR; + || !resource.getTsFileRepairStatus().isNormalCompactionCandidate(); } /** @@ -188,7 +187,7 @@ public List selectInnerSpaceTask(List try { // 1. select compaction task based on file which need to repair List taskList = selectFileNeedToRepair(); - if (!taskList.isEmpty()) { + if (!taskList.isEmpty() || !CompactionUtils.isDiskHasSpace()) { return taskList; } // 2. if a suitable compaction task is not selected in the first step, select the compaction @@ -213,10 +212,13 @@ protected List selectTaskBaseOnLevel() } private List selectFileNeedToRepair() { + if (!config.isEnableAutoRepairCompaction()) { + return Collections.emptyList(); + } List taskList = new ArrayList<>(); for (TsFileResource resource : tsFileResources) { if (resource.getStatus() == TsFileResourceStatus.NORMAL - && resource.getTsFileRepairStatus() == TsFileRepairStatus.NEED_TO_REPAIR) { + && resource.getTsFileRepairStatus().isRepairCompactionCandidate()) { taskList.add( new RepairUnsortedFileCompactionTask( timePartition, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileRepairStatus.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileRepairStatus.java index 933551d227b9..72ef7f816fed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileRepairStatus.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileRepairStatus.java @@ -21,6 +21,18 @@ public enum TsFileRepairStatus { NORMAL, - NEED_TO_REPAIR, - CAN_NOT_REPAIR + NEED_TO_CHECK, + NEED_TO_REPAIR_BY_REWRITE, + NEED_TO_REPAIR_BY_MOVE, + CAN_NOT_REPAIR; + + public boolean isNormalCompactionCandidate() { + return this == NORMAL; + } + + public boolean isRepairCompactionCandidate() { + return this == NEED_TO_CHECK + || this == NEED_TO_REPAIR_BY_REWRITE + || this == NEED_TO_REPAIR_BY_MOVE; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index fd83f4973014..27e81ac93ebe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -648,7 +648,7 @@ public void moveTo(File targetDir) throws IOException { @Override public String toString() { - return String.format("file is %s, status: %s", file.toString(), getStatus()); + return String.format("{file: %s, status: %s}", file.toString(), getStatus()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheWriter.java index 3d88575ba683..30a8a24f4b78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/fileTimeIndexCache/FileTimeIndexCacheWriter.java @@ -57,9 +57,11 @@ public void write(ByteBuffer logBuffer) throws IOException { // For UT env, logFile may not be created return; } - channel.write(logBuffer); - if (this.forceEachWrite) { - channel.force(true); + if (channel != null && channel.isOpen()) { + channel.write(logBuffer); + if (this.forceEachWrite) { + channel.force(true); + } } } catch (ClosedChannelException ignored) { logger.warn("someone interrupt current thread, so no need to do write for io safety"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java index e952dec11791..8b7f1f6336dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALManager.java @@ -169,7 +169,7 @@ private void deleteOutdatedFiles() { deleteOutdatedFilesInWALNodes(); if (firstLoop && shouldThrottle()) { logger.warn( - "WAL disk usage {} is larger than the iot_consensus_throttle_threshold_in_byte * 0.8 {}, please check your write load, iot consensus and the pipe module. It's better to allocate more disk for WAL.", + "WAL disk usage {} is larger than the wal_throttle_threshold_in_byte * 0.8 {}, please check your write load, iot consensus and the pipe module. It's better to allocate more disk for WAL.", getTotalDiskUsage(), getThrottleThreshold()); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java index 7f4f187ebd78..13770fdbf7c2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/agent/SubscriptionBrokerAgent.java @@ -203,4 +203,14 @@ public void executePrefetch(final String consumerGroupId, final String topicName } broker.executePrefetch(topicName); } + + public int getPipeEventCount(final String consumerGroupId, final String topicName) { + final SubscriptionBroker broker = consumerGroupIdToSubscriptionBroker.get(consumerGroupId); + if (Objects.isNull(broker)) { + LOGGER.warn( + "Subscription: broker bound to consumer group [{}] does not exist", consumerGroupId); + return 0; + } + return broker.getPipeEventCount(topicName); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java index f6fc757f5ceb..420b571103a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionBroker.java @@ -368,4 +368,24 @@ public void executePrefetch(final String topicName) { } prefetchingQueue.executePrefetch(); } + + public int getPipeEventCount(final String topicName) { + final SubscriptionPrefetchingQueue prefetchingQueue = + topicNameToPrefetchingQueue.get(topicName); + if (Objects.isNull(prefetchingQueue)) { + LOGGER.warn( + "Subscription: prefetching queue bound to topic [{}] for consumer group [{}] does not exist", + topicName, + brokerId); + return 0; + } + if (prefetchingQueue.isClosed()) { + LOGGER.warn( + "Subscription: prefetching queue bound to topic [{}] for consumer group [{}] is closed", + topicName, + brokerId); + return 0; + } + return prefetchingQueue.getPipeEventCount(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java index e23f5fb05275..ac5b3fad279c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingQueue.java @@ -531,7 +531,7 @@ public static String generatePrefetchingQueueId( return consumerGroupId + "_" + topicName; } - public long getUncommittedEventCount() { + public long getSubscriptionUncommittedEventCount() { return inFlightEvents.size(); } @@ -539,6 +539,18 @@ public long getCurrentCommitId() { return commitIdGenerator.get(); } + public int getPipeEventCount() { + return inputPendingQueue.size() + + prefetchingQueue.stream() + .map(SubscriptionEvent::getPipeEventCount) + .reduce(Integer::sum) + .orElse(0) + + +inFlightEvents.values().stream() + .map(SubscriptionEvent::getPipeEventCount) + .reduce(Integer::sum) + .orElse(0); + } + /////////////////////////////// close & termination /////////////////////////////// public boolean isClosed() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java index 2f7902f1c6b1..ad274023f6a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEvent.java @@ -415,6 +415,12 @@ public String getFileName() { return pipeEvents.getTsFile().getName(); } + /////////////////////////////// APIs provided for metric framework /////////////////////////////// + + public int getPipeEventCount() { + return pipeEvents.getPipeEventCount(); + } + /////////////////////////////// object /////////////////////////////// @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java index 54363ac8c7cb..871b1c12587f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTabletEventBatch.java @@ -70,7 +70,7 @@ public SubscriptionPipeTabletEventBatch( @Override public synchronized boolean onEvent(final Consumer consumer) { - if (shouldEmit()) { + if (shouldEmit() && !enrichedEvents.isEmpty()) { if (Objects.isNull(events)) { events = generateSubscriptionEvents(); } @@ -98,6 +98,8 @@ public synchronized void cleanUp() { for (final EnrichedEvent enrichedEvent : enrichedEvents) { enrichedEvent.clearReferenceCount(this.getClass().getName()); } + enrichedEvents.clear(); + tablets.clear(); } public synchronized void ack() { @@ -221,4 +223,10 @@ private static String formatEnrichedEvents( } return eventMessageList.toString(); } + + //////////////////////////// APIs provided for metric framework //////////////////////////// + + public int getPipeEventCount() { + return enrichedEvents.size(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java index 2b5e29f7f6c8..e38887e19f2f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/batch/SubscriptionPipeTsFileEventBatch.java @@ -43,6 +43,7 @@ public class SubscriptionPipeTsFileEventBatch extends SubscriptionPipeEventBatch { private final PipeTabletEventTsFileBatch batch; + private final List enrichedEvents; public SubscriptionPipeTsFileEventBatch( final int regionId, @@ -51,11 +52,12 @@ public SubscriptionPipeTsFileEventBatch( final long maxBatchSizeInBytes) { super(regionId, prefetchingQueue, maxDelayInMs, maxBatchSizeInBytes); this.batch = new PipeTabletEventTsFileBatch(maxDelayInMs, maxBatchSizeInBytes); + this.enrichedEvents = new ArrayList<>(); } @Override public synchronized boolean onEvent(final Consumer consumer) throws Exception { - if (batch.shouldEmit()) { + if (batch.shouldEmit() && !enrichedEvents.isEmpty()) { if (Objects.isNull(events)) { events = generateSubscriptionEvents(); } @@ -74,6 +76,7 @@ public synchronized boolean onEvent( throws Exception { if (event instanceof TabletInsertionEvent) { batch.onEvent((TabletInsertionEvent) event); // no exceptions will be thrown + enrichedEvents.add(event); event.decreaseReferenceCount( SubscriptionPipeTsFileEventBatch.class.getName(), false); // missing releaseLastEvent decreases reference count @@ -85,6 +88,7 @@ public synchronized boolean onEvent( public synchronized void cleanUp() { // close batch, it includes clearing the reference count of events batch.close(); + enrichedEvents.clear(); } public synchronized void ack() { @@ -128,4 +132,10 @@ protected Map coreReportMessage() { coreReportMessage.put("batch", batch.toString()); return coreReportMessage; } + + //////////////////////////// APIs provided for metric framework //////////////////////////// + + public int getPipeEventCount() { + return enrichedEvents.size(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeEmptyEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeEmptyEvent.java index d0556cd15b7f..3e74e03d8f4c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeEmptyEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeEmptyEvent.java @@ -34,8 +34,17 @@ public void ack() {} @Override public void cleanUp() {} + /////////////////////////////// stringify /////////////////////////////// + @Override public String toString() { return "SubscriptionEmptyPipeEvent"; } + + //////////////////////////// APIs provided for metric framework //////////////////////////// + + @Override + public int getPipeEventCount() { + return 0; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeEvents.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeEvents.java index 813690dd00b0..489c4cf8120c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeEvents.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeEvents.java @@ -31,4 +31,8 @@ public interface SubscriptionPipeEvents { void ack(); void cleanUp(); + + //////////////////////////// APIs provided for metric framework //////////////////////////// + + int getPipeEventCount(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTabletBatchEvents.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTabletBatchEvents.java index fb9e31f77c2b..226367405eba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTabletBatchEvents.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTabletBatchEvents.java @@ -46,8 +46,17 @@ public void cleanUp() { batch.cleanUp(); } + /////////////////////////////// stringify /////////////////////////////// + @Override public String toString() { return "SubscriptionPipeTabletBatchEvents{batch=" + batch + "}"; } + + //////////////////////////// APIs provided for metric framework //////////////////////////// + + @Override + public int getPipeEventCount() { + return batch.getPipeEventCount(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFileBatchEvents.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFileBatchEvents.java index aa9a3593a93b..5cae21f5ac8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFileBatchEvents.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFileBatchEvents.java @@ -29,6 +29,7 @@ public class SubscriptionPipeTsFileBatchEvents implements SubscriptionPipeEvents private final SubscriptionPipeTsFileEventBatch batch; private final File tsFile; private final AtomicInteger referenceCount; // shared between the same batch + private final int count; // snapshot the initial reference count, used for event count calculation public SubscriptionPipeTsFileBatchEvents( final SubscriptionPipeTsFileEventBatch batch, @@ -37,6 +38,7 @@ public SubscriptionPipeTsFileBatchEvents( this.batch = batch; this.tsFile = tsFile; this.referenceCount = referenceCount; + this.count = Math.max(1, referenceCount.get()); } @Override @@ -58,6 +60,8 @@ public void cleanUp() { } } + /////////////////////////////// stringify /////////////////////////////// + @Override public String toString() { return "SubscriptionPipeTsFileBatchEvents{batch=" @@ -68,4 +72,13 @@ public String toString() { + referenceCount + "}"; } + + //////////////////////////// APIs provided for metric framework //////////////////////////// + + @Override + public int getPipeEventCount() { + // Since multiple events will share the same batch, equal division is performed here. + // If it is not exact, round up to remain pessimistic. + return (batch.getPipeEventCount() + count - 1) / count; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFilePlainEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFilePlainEvent.java index c210635a0615..111006fa6d32 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFilePlainEvent.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/pipe/SubscriptionPipeTsFilePlainEvent.java @@ -47,10 +47,19 @@ public void cleanUp() { tsFileInsertionEvent.clearReferenceCount(this.getClass().getName()); } + /////////////////////////////// stringify /////////////////////////////// + @Override public String toString() { return "SubscriptionPipeTsFilePlainEvent{tsFileInsertionEvent=" + tsFileInsertionEvent.coreReportMessage() + "}"; } + + //////////////////////////// APIs provided for metric framework //////////////////////////// + + @Override + public int getPipeEventCount() { + return 1; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/metric/SubscriptionPrefetchingQueueMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/metric/SubscriptionPrefetchingQueueMetrics.java index d38de0d709e1..4e712fb29722 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/metric/SubscriptionPrefetchingQueueMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/metric/SubscriptionPrefetchingQueueMetrics.java @@ -92,7 +92,7 @@ private void createAutoGauge(final String id) { Metric.SUBSCRIPTION_UNCOMMITTED_EVENT_COUNT.toString(), MetricLevel.IMPORTANT, queue, - SubscriptionPrefetchingQueue::getUncommittedEventCount, + SubscriptionPrefetchingQueue::getSubscriptionUncommittedEventCount, Tag.NAME.toString(), queue.getPrefetchingQueueId()); // current commit id diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java index 8524e77d040e..3e540254d8e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtask.java @@ -25,8 +25,13 @@ import org.apache.iotdb.pipe.api.PipeConnector; import org.apache.iotdb.pipe.api.event.Event; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class SubscriptionConnectorSubtask extends PipeConnectorSubtask { + private static final Logger LOGGER = LoggerFactory.getLogger(SubscriptionConnectorSubtask.class); + private final String topicName; private final String consumerGroupId; @@ -72,4 +77,13 @@ public String getConsumerGroupId() { public UnboundedBlockingPendingQueue getInputPendingQueue() { return inputPendingQueue; } + + //////////////////////////// APIs provided for metric framework //////////////////////////// + + @Override + public int getEventCount(final String pipeName) { + // count the number of pipe events in sink queue and prefetching queue, note that can safely + // ignore lastEvent + return SubscriptionAgent.broker().getPipeEventCount(consumerGroupId, topicName); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskManager.java index dfce62faac8b..e0707ebf5473 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/task/subtask/SubscriptionConnectorSubtaskManager.java @@ -31,6 +31,7 @@ import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeConnectorSubtask; import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeConnectorSubtaskLifeCycle; import org.apache.iotdb.db.pipe.agent.task.subtask.connector.PipeRealtimePriorityBlockingQueue; +import org.apache.iotdb.db.pipe.metric.PipeDataNodeRemainingEventAndTimeMetrics; import org.apache.iotdb.db.pipe.metric.PipeDataRegionEventCounter; import org.apache.iotdb.pipe.api.PipeConnector; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; @@ -157,6 +158,8 @@ public synchronized String register( final PipeConnectorSubtaskLifeCycle lifeCycle = attributeSortedString2SubtaskLifeCycleMap.get(attributeSortedString); lifeCycle.register(); + PipeDataNodeRemainingEventAndTimeMetrics.getInstance() + .register(lifeCycle.getSubtask(), environment.getPipeName(), environment.getCreationTime()); return attributeSortedString; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java index ba3a4d86d722..541bf90ec85f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/constant/SqlConstant.java @@ -50,11 +50,16 @@ protected SqlConstant() { public static final String MAX_TIME = "max_time"; public static final String MAX_VALUE = "max_value"; public static final String MIN_VALUE = "min_value"; + public static final String MAX = "max"; + public static final String MIN = "min"; public static final String MAX_BY = "max_by"; public static final String MIN_BY = "min_by"; public static final String EXTREME = "extreme"; public static final String FIRST_VALUE = "first_value"; public static final String LAST_VALUE = "last_value"; + public static final String FIRST = "first"; + public static final String FIRST_BY = "first_by"; + public static final String LAST_BY = "last_by"; public static final String COUNT = "count"; public static final String AVG = "avg"; public static final String SUM = "sum"; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationMergeSortOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationMergeSortOperatorTest.java new file mode 100644 index 000000000000..4d9edecf00cf --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/AggregationMergeSortOperatorTest.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator; + +import org.apache.iotdb.db.queryengine.execution.aggregation.CountAccumulator; +import org.apache.iotdb.db.queryengine.execution.driver.DriverContext; +import org.apache.iotdb.db.queryengine.execution.operator.process.AggregationMergeSortOperator; +import org.apache.iotdb.db.queryengine.execution.operator.process.ProcessOperator; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.queryengine.plan.statement.component.SortItem; +import org.apache.iotdb.db.utils.datastructure.SortKey; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.read.common.block.column.BinaryColumn; +import org.apache.tsfile.read.common.block.column.LongColumn; +import org.apache.tsfile.utils.Binary; +import org.junit.Test; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; + +import static org.apache.iotdb.db.queryengine.execution.operator.process.join.merge.MergeSortComparator.getComparator; +import static org.junit.Assert.assertEquals; + +public class AggregationMergeSortOperatorTest { + + @Test + public void deviceInTwoRegionTest() throws Exception { + OperatorContext operatorContext = + new OperatorContext(1, new PlanNodeId("1"), "test-type", new DriverContext()); + + MockDeviceViewOperator1 operator1 = new MockDeviceViewOperator1(operatorContext); + MockDeviceViewOperator1 operator2 = new MockDeviceViewOperator2(operatorContext); + + List sortItemList = + Arrays.asList(new SortItem("DEVICE", Ordering.ASC), new SortItem("TIME", Ordering.ASC)); + List sortItemIndexList = Arrays.asList(0, -1); + List sortItemDataTypeList = Arrays.asList(TSDataType.TEXT, TSDataType.INT64); + Comparator comparator = + getComparator(sortItemList, sortItemIndexList, sortItemDataTypeList); + + AggregationMergeSortOperator operator = + new AggregationMergeSortOperator( + operatorContext, + Arrays.asList(operator1, operator2), + Arrays.asList(TSDataType.TEXT, TSDataType.INT64), + Collections.singletonList(new CountAccumulator()), + false, + comparator); + int cnt = 0; + while (operator.isBlocked().isDone() && operator.hasNext()) { + TsBlock block = operator.next(); + if (block != null && block.getPositionCount() > 0) { + if (cnt == 0) { + assertEquals("d1", block.getColumn(0).getBinary(0).toString()); + assertEquals(3, block.getColumn(1).getLong(0)); + } else { + assertEquals("d2", block.getColumn(0).getBinary(0).toString()); + assertEquals(5, block.getColumn(1).getLong(0)); + } + cnt++; + } + } + assertEquals(2, cnt); + } + + private static class MockDeviceViewOperator1 implements ProcessOperator { + + OperatorContext operatorContext; + int invokeCount = 0; + + public MockDeviceViewOperator1(OperatorContext operatorContext) { + this.operatorContext = operatorContext; + } + + @Override + public OperatorContext getOperatorContext() { + return operatorContext; + } + + @Override + public TsBlock next() throws Exception { + if (invokeCount == 0) { + invokeCount++; + return buildTsBlock("d1", 1); + } + return null; + } + + @Override + public boolean hasNext() throws Exception { + return invokeCount < 1; + } + + @Override + public void close() throws Exception {} + + @Override + public boolean isFinished() throws Exception { + return invokeCount < 1; + } + + @Override + public long calculateMaxPeekMemory() { + return 0; + } + + @Override + public long calculateMaxReturnSize() { + return 0; + } + + @Override + public long calculateRetainedSizeAfterCallingNext() { + return 0; + } + + @Override + public long ramBytesUsed() { + return 0; + } + } + + private static class MockDeviceViewOperator2 extends MockDeviceViewOperator1 { + + public MockDeviceViewOperator2(OperatorContext operatorContext) { + super(operatorContext); + } + + @Override + public TsBlock next() throws Exception { + if (invokeCount == 0) { + invokeCount++; + return buildTsBlock("d1", 2); + } else if (invokeCount == 1) { + invokeCount++; + return buildTsBlock("d2", 5); + } + return null; + } + + @Override + public boolean hasNext() throws Exception { + return invokeCount < 2; + } + } + + private static TsBlock buildTsBlock(String device, int count) { + LongColumn timeColumn = new LongColumn(1, Optional.empty(), new long[] {0}); + BinaryColumn deviceColumn = + new BinaryColumn(1, Optional.empty(), new Binary[] {new Binary(device.getBytes())}); + LongColumn countColumn = new LongColumn(1, Optional.empty(), new long[] {count}); + return new TsBlock(1, timeColumn, deviceColumn, countColumn); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationTest.java index c8cdb87ce7dd..2aa5158060fe 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/AggregationTest.java @@ -49,6 +49,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.singleGroupingSet; import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.tableScan; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.FINAL; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.INTERMEDIATE; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.PARTIAL; import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode.Step.SINGLE; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticBinaryExpression.Operator.ADD; @@ -387,6 +388,96 @@ public void partialPushDownTest() { "testdb.table1", ImmutableList.of("attr1", "tag1", "date_bin$gid", "count_0"), ImmutableSet.of("attr1", "tag1", "time", "s2")))))); + + // GlobalAggregation with more than one DeviceEntry + // Output - Aggregation - AggTableScan + assertPlan( + planTester.createPlan("SELECT count(s2) FROM table1"), + output( + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.empty(), aggregationFunction("count", ImmutableList.of("count_0"))), + ImmutableList.of(), + Optional.empty(), + FINAL, + aggregationTableScan( + singleGroupingSet(), + ImmutableList.of(), // UnStreamable + Optional.empty(), + PARTIAL, + "testdb.table1", + ImmutableList.of("count_0"), + ImmutableSet.of("s2"))))); + + // Output - Aggregation(FINAL) - Collect - Aggregation(INTERMEDIATE) - AggTableScan + assertPlan( + planTester.getFragmentPlan(0), + output( + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.empty(), aggregationFunction("count", ImmutableList.of("count_1"))), + ImmutableList.of(), + Optional.empty(), + FINAL, + collect( + exchange(), + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("count_1"), + aggregationFunction("count", ImmutableList.of("count_0"))), + ImmutableList.of(), + Optional.empty(), + INTERMEDIATE, + aggregationTableScan( + singleGroupingSet(), + ImmutableList.of(), // UnStreamable + Optional.empty(), + PARTIAL, + "testdb.table1", + ImmutableList.of("count_0"), + ImmutableSet.of("s2"))), + exchange())))); + + // - Aggregation(INTERMEDIATE) - AggTableScan + assertPlan( + planTester.getFragmentPlan(1), + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("count_1"), aggregationFunction("count", ImmutableList.of("count_0"))), + ImmutableList.of(), + Optional.empty(), + INTERMEDIATE, + aggregationTableScan( + singleGroupingSet(), + ImmutableList.of(), // UnStreamable + Optional.empty(), + PARTIAL, + "testdb.table1", + ImmutableList.of("count_0"), + ImmutableSet.of("s2")))); + + // - Aggregation(INTERMEDIATE) - AggTableScan + assertPlan( + planTester.getFragmentPlan(2), + aggregation( + singleGroupingSet(), + ImmutableMap.of( + Optional.of("count_1"), aggregationFunction("count", ImmutableList.of("count_0"))), + ImmutableList.of(), + Optional.empty(), + INTERMEDIATE, + aggregationTableScan( + singleGroupingSet(), + ImmutableList.of(), // UnStreamable + Optional.empty(), + PARTIAL, + "testdb.table1", + ImmutableList.of("count_0"), + ImmutableSet.of("s2")))); } @Test @@ -533,52 +624,62 @@ public void completePushDownTest() { "testdb.table1", ImmutableList.of("tag1", "tag2", "tag3", "attr1", "time", "count"), ImmutableSet.of("tag1", "tag2", "tag3", "attr1", "time", "s2"))))); + } + + @Test + public void syntacticSugarTest() { + PlanTester planTester = new PlanTester(); - // GlobalAggregation + // First and last need to be added the second argument 'time' if it is not explicit + LogicalQueryPlan logicalQueryPlan = + planTester.createPlan("SELECT first(s1+1), last(s2) FROM table1"); assertPlan( - planTester.createPlan("SELECT count(s2) FROM table1"), + logicalQueryPlan, output( - aggregationTableScan( + aggregation( singleGroupingSet(), - ImmutableList.of(), // UnStreamable + ImmutableMap.of( + Optional.of("first"), + aggregationFunction("first", ImmutableList.of("expr", "time")), + Optional.of("last"), + aggregationFunction("last", ImmutableList.of("s2", "time"))), + ImmutableList.of(), Optional.empty(), SINGLE, - "testdb.table1", - ImmutableList.of("count"), - ImmutableSet.of("s2")))); + project( + ImmutableMap.of( + "expr", + expression( + new ArithmeticBinaryExpression( + ADD, new SymbolReference("s1"), new LongLiteral("1")))), + tableScan( + "testdb.table1", + ImmutableList.of("time", "s1", "s2"), + ImmutableSet.of("s1", "s2", "time")))))); + } + + @Test + public void withTimePushDownLevelTest() { + PlanTester planTester = new PlanTester(); + + // first, last, first_by with time, last_by with time should be push-down + LogicalQueryPlan logicalQueryPlan = + planTester.createPlan( + "SELECT first(s1), last(s1), first_by(time,s1), last_by(time,s1) FROM table1 group by tag1, tag2, tag3"); + + // Output - Project - AggregationTableScan assertPlan( - planTester.getFragmentPlan(0), + logicalQueryPlan, output( - collect( - exchange(), + project( aggregationTableScan( - singleGroupingSet(), - ImmutableList.of(), // UnStreamable + singleGroupingSet("tag1", "tag2", "tag3"), + ImmutableList.of("tag1", "tag2", "tag3"), // Streamable Optional.empty(), SINGLE, "testdb.table1", - ImmutableList.of("count"), - ImmutableSet.of("s2")), - exchange()))); - assertPlan( - planTester.getFragmentPlan(1), - aggregationTableScan( - singleGroupingSet(), - ImmutableList.of(), // UnStreamable - Optional.empty(), - SINGLE, - "testdb.table1", - ImmutableList.of("count"), - ImmutableSet.of("s2"))); - assertPlan( - planTester.getFragmentPlan(2), - aggregationTableScan( - singleGroupingSet(), - ImmutableList.of(), // UnStreamable - Optional.empty(), - SINGLE, - "testdb.table1", - ImmutableList.of("count"), - ImmutableSet.of("s2"))); + ImmutableList.of( + "tag1", "tag2", "tag3", "first", "last", "first_by", "last_by"), + ImmutableSet.of("tag1", "tag2", "tag3", "s1", "time"))))); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java index fdcf49a91423..13652a7c9c7f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/JoinTest.java @@ -385,15 +385,7 @@ public void unsupportedJoinTest() { "SELECT * FROM table1 t1 INNER JOIN table1 t2 USING(tag1, time)", ONLY_SUPPORT_TIME_COLUMN_IN_USING_CLAUSE); - // FULL, LEFT, RIGHT JOIN - assertAnalyzeSemanticException( - "SELECT * FROM table1 t1 FULL JOIN table1 t2 ON t1.time=t2.time", - "FULL JOIN is not supported, only support INNER JOIN in current version"); - - assertAnalyzeSemanticException( - "SELECT * FROM table1 t1 FULL JOIN table1 t2 ON t1.time=t2.time WHERE t1.time>1", - "FULL JOIN is not supported, only support INNER JOIN in current version"); - + // LEFT, RIGHT JOIN assertAnalyzeSemanticException( "SELECT * FROM table1 t1 LEFT JOIN table1 t2 ON t1.time=t2.time", "LEFT JOIN is not supported, only support INNER JOIN in current version"); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java index 209bdf86f1b8..244a1a8f4965 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSMetadata.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.OperatorNotFoundException; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableBuiltinAggregationFunction; import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@ -364,8 +365,8 @@ public DataPartition getDataPartitionWithUnclosedTimeRange( } @Override - public boolean canUseStatistics(String name) { - return BuiltinAggregationFunction.canUseStatistics(name); + public boolean canUseStatistics(String name, boolean withTime) { + return TableBuiltinAggregationFunction.canUseStatistics(name, withTime); } private static final DataPartition DATA_PARTITION = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSTest.java index 9e0bd1a27415..472393b7ba3e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TSBSTest.java @@ -60,6 +60,7 @@ import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression.Operator.LESS_THAN; import static org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.TableBuiltinScalarFunction.DATE_BIN; +@Ignore // TODO public class TSBSTest { private static final PlanTester planTester = new PlanTester(new TSBSMetadata()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java index f0ab196e4d13..d6c7d1b15ba7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/TestMatadata.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.OperatorNotFoundException; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableBuiltinAggregationFunction; import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ComparisonExpression; @@ -311,8 +312,8 @@ public DataPartition getDataPartitionWithUnclosedTimeRange( } @Override - public boolean canUseStatistics(String name) { - return BuiltinAggregationFunction.canUseStatistics(name); + public boolean canUseStatistics(String name, boolean withTime) { + return TableBuiltinAggregationFunction.canUseStatistics(name, withTime); } private static final DataPartition DATA_PARTITION = diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AggregationTableScanMatcher.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AggregationTableScanMatcher.java index 96b7f98d45f9..cc15d2bbbe08 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AggregationTableScanMatcher.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/AggregationTableScanMatcher.java @@ -20,6 +20,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; +import com.google.common.collect.ImmutableSet; + import java.util.Collection; import java.util.List; import java.util.Optional; @@ -80,10 +82,11 @@ public MatchResult detailMatches( } if (!outputSymbols.isEmpty() - && !outputSymbols.equals( - aggregationTableScanNode.getOutputSymbols().stream() - .map(Symbol::getName) - .collect(Collectors.toList()))) { + && !ImmutableSet.copyOf(outputSymbols) + .equals( + aggregationTableScanNode.getOutputSymbols().stream() + .map(Symbol::getName) + .collect(Collectors.toSet()))) { return NO_MATCH; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index 2d1bac2136f7..8fa0df3f7342 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -256,7 +256,7 @@ public static ExpectedValueProvider aggregationFunction( } // Attention: Now we only pass aliases according to outputSymbols, but we don't verify the output - // column if exists in Table because there maybe partial Agg-result. + // column if exists in Table and their order because there maybe partial Agg-result. public static PlanMatchPattern aggregationTableScan( GroupingSetDescriptor groupingSets, List preGroupedSymbols, diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java index 9e58259d722b..18c3f9bd5c1c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileCompactionTest.java @@ -112,6 +112,7 @@ public void testRepairUnsortedDataBetweenPageWithNonAlignedSeries() throws IOExc writer.endFile(); } Assert.assertFalse(TsFileResourceUtils.validateTsFileDataCorrectness(resource)); + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0, true); task.start(); @@ -136,6 +137,7 @@ public void testRepairUnsortedDataBetweenPageWithAlignedSeries() throws IOExcept writer.endFile(); } Assert.assertFalse(TsFileResourceUtils.validateTsFileDataCorrectness(resource)); + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0, true); task.start(); @@ -164,6 +166,7 @@ public void testRepairUnsortedDataInOnePageWithNonAlignedSeries() throws IOExcep writer.endFile(); } Assert.assertFalse(TsFileResourceUtils.validateTsFileDataCorrectness(resource)); + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0, true); task.start(); @@ -196,6 +199,7 @@ public void testRepairUnsortedDataInOnePageWithMultiNonAlignedSeries() throws IO writer.endFile(); } Assert.assertFalse(TsFileResourceUtils.validateTsFileDataCorrectness(resource)); + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0, true); task.start(); @@ -224,6 +228,7 @@ public void testRepairUnsortedDataInOnePageWithUnseqFile() throws IOException { writer.endFile(); } Assert.assertFalse(TsFileResourceUtils.validateTsFileDataCorrectness(resource)); + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0, true); task.start(); @@ -252,6 +257,7 @@ public void testRepairUnsortedDataInOnePageWithAlignedSeries() throws IOExceptio writer.endFile(); } Assert.assertFalse(TsFileResourceUtils.validateTsFileDataCorrectness(resource)); + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); RepairUnsortedFileCompactionTask task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, resource.isSeq(), 0, true); task.start(); @@ -307,7 +313,7 @@ public void testMarkFileAndRepairWithInnerSeqSpaceCompactionTask() Assert.assertFalse(task.start()); for (TsFileResource resource : tsFileManager.getTsFileList(true)) { - Assert.assertEquals(resource.getTsFileRepairStatus(), TsFileRepairStatus.NEED_TO_REPAIR); + Assert.assertTrue(resource.getTsFileRepairStatus().isRepairCompactionCandidate()); } long initialFinishedCompactionTaskNum = @@ -382,7 +388,7 @@ public void testMarkFileAndRepairWithInnerUnSeqSpaceCompactionTask() Assert.assertFalse(task.start()); for (TsFileResource resource : tsFileManager.getTsFileList(true)) { - Assert.assertEquals(resource.getTsFileRepairStatus(), TsFileRepairStatus.NEED_TO_REPAIR); + Assert.assertTrue(resource.getTsFileRepairStatus().isRepairCompactionCandidate()); } long initialFinishedCompactionTaskNum = @@ -458,7 +464,7 @@ public void testMarkFileAndRepairWithCrossSpaceCompactionTask() Assert.assertFalse(task.start()); for (TsFileResource resource : tsFileManager.getTsFileList(true)) { - Assert.assertEquals(resource.getTsFileRepairStatus(), TsFileRepairStatus.NEED_TO_REPAIR); + Assert.assertTrue(resource.getTsFileRepairStatus().isRepairCompactionCandidate()); } long initialFinishedCompactionTaskNum = @@ -521,8 +527,9 @@ public void testRepairOverlapBetweenFile() throws IOException { tsFileManager.addAll(seqResources, true); Assert.assertFalse(TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(seqResources)); + seqResource2.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_MOVE); RepairUnsortedFileCompactionTask task = - new RepairUnsortedFileCompactionTask(0, tsFileManager, seqResource2, true, false, 0, true); + new RepairUnsortedFileCompactionTask(0, tsFileManager, seqResource2, true, 0, true); Assert.assertTrue(task.start()); Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); @@ -571,8 +578,9 @@ public void testRepairOverlapBetweenFileWithModFile() throws IOException, Illega tsFileManager.addAll(seqResources, true); Assert.assertFalse(TsFileResourceUtils.validateTsFileResourcesHasNoOverlap(seqResources)); + seqResource2.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_MOVE); RepairUnsortedFileCompactionTask task = - new RepairUnsortedFileCompactionTask(0, tsFileManager, seqResource2, true, false, 0, true); + new RepairUnsortedFileCompactionTask(0, tsFileManager, seqResource2, true, 0, true); Assert.assertTrue(task.start()); Assert.assertEquals(1, tsFileManager.getTsFileList(true).size()); Assert.assertEquals(1, tsFileManager.getTsFileList(false).size()); @@ -630,10 +638,17 @@ public void testEstimateRepairCompactionMemory() throws IOException { writer.endChunkGroup(); writer.endFile(); } + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); RepairUnsortedFileCompactionTask task = - new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, true, 0, true); + new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, 0, true); Assert.assertTrue(task.getEstimatedMemoryCost() > 0); - task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, false, 0, true); + + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_CHECK); + task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, false, 0, true); + Assert.assertTrue(task.getEstimatedMemoryCost() > 0); + + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_MOVE); + task = new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, false, 0, true); Assert.assertEquals(0, task.getEstimatedMemoryCost()); } @@ -657,8 +672,9 @@ public void testMergeAlignedSeriesPointWithSameTimestamp() throws IOException { writer.endChunkGroup(); writer.endFile(); } + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); RepairUnsortedFileCompactionTask task = - new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, true, 0, true); + new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, 0, true); Assert.assertTrue(task.start()); TsFileResource target = tsFileManager.getTsFileList(false).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { @@ -691,6 +707,71 @@ public void testMergeAlignedSeriesPointWithSameTimestamp() throws IOException { } } + @Test + public void testRepairFilesWithCheck1() throws IOException { + TsFileResource resource = createEmptyFileAndResource(true); + try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) { + writer.startChunkGroup("d1"); + writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue( + Arrays.asList("s1", "s2", "s3"), + new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20)}}, + TSEncoding.PLAIN, + CompressionType.LZ4, + Arrays.asList(true, false, false)); + writer.endChunkGroup(); + writer.endFile(); + } + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_CHECK); + RepairUnsortedFileCompactionTask task = + new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, 0, true); + Assert.assertTrue(task.start()); + Assert.assertEquals( + resource.getTsFileRepairStatus(), TsFileRepairStatus.NEED_TO_REPAIR_BY_MOVE); + } + + @Test + public void testRepairFilesWithCheck2() throws IOException { + TsFileResource resource = createEmptyFileAndResource(true); + try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) { + writer.startChunkGroup("d1"); + writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue( + Arrays.asList("s1", "s2", "s3"), + new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20), new TimeRange(1, 10)}}, + TSEncoding.PLAIN, + CompressionType.LZ4, + Arrays.asList(true, false, false)); + writer.endChunkGroup(); + writer.endFile(); + } + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_CHECK); + RepairUnsortedFileCompactionTask task = + new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, 0, true); + Assert.assertTrue(task.start()); + Assert.assertEquals( + resource.getTsFileRepairStatus(), TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); + } + + @Test + public void testRepairFilesWithCheck3() throws IOException { + TsFileResource resource = createEmptyFileAndResource(false); + try (CompactionTestFileWriter writer = new CompactionTestFileWriter(resource)) { + writer.startChunkGroup("d1"); + writer.generateSimpleAlignedSeriesToCurrentDeviceWithNullValue( + Arrays.asList("s1", "s2", "s3"), + new TimeRange[][] {new TimeRange[] {new TimeRange(10, 20)}}, + TSEncoding.PLAIN, + CompressionType.LZ4, + Arrays.asList(true, false, false)); + writer.endChunkGroup(); + writer.endFile(); + } + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_CHECK); + RepairUnsortedFileCompactionTask task = + new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, 0, true); + Assert.assertTrue(task.start()); + Assert.assertEquals(resource.getTsFileRepairStatus(), TsFileRepairStatus.NORMAL); + } + @Test public void testSplitChunk() throws IOException { TsFileResource resource = createEmptyFileAndResource(true); @@ -705,8 +786,9 @@ public void testSplitChunk() throws IOException { writer.endChunkGroup(); writer.endFile(); } + resource.setTsFileRepairStatus(TsFileRepairStatus.NEED_TO_REPAIR_BY_REWRITE); RepairUnsortedFileCompactionTask task = - new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, true, 0, true); + new RepairUnsortedFileCompactionTask(0, tsFileManager, resource, true, 0, true); Assert.assertTrue(task.start()); TsFileResource target = tsFileManager.getTsFileList(false).get(0); try (TsFileSequenceReader reader = new TsFileSequenceReader(target.getTsFilePath())) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileSchedulerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileSchedulerTest.java index 646626fb8c4f..92337fe151cd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileSchedulerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/repair/RepairUnsortedFileSchedulerTest.java @@ -198,7 +198,7 @@ public void testRecoverRepairScheduleSkipRepairedTimePartitionAndMarkFile() thro scheduler.run(); Assert.assertEquals(3, tsFileManager.getTsFileList(true).size()); // check whether the repair status is marked correctly - Assert.assertEquals(TsFileRepairStatus.NEED_TO_REPAIR, seqResource3.getTsFileRepairStatus()); + Assert.assertTrue(seqResource3.getTsFileRepairStatus().isRepairCompactionCandidate()); } @Test diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java index 641933f70a5a..651caadb1504 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/CompactionWithAllNullRowsTest.java @@ -73,6 +73,7 @@ public void setUp() IoTDBDescriptor.getInstance() .getConfig() .setCompactionMaxAlignedSeriesNumInOneBatch(maxAlignedSeriesNumInOneBatch); + super.setUp(); } @After @@ -84,6 +85,7 @@ public void tearDown() throws IOException, StorageEngineException { IoTDBDescriptor.getInstance() .getConfig() .setCompactionMaxAlignedSeriesNumInOneBatch(defaultMaxAlignedSeriesNumInOneBatch); + super.tearDown(); } @Parameterized.Parameters(name = "type={0} batch_size={1}") diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java index 5755964a4366..ef804437361b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/BatchCompactionUtilsTest.java @@ -62,6 +62,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; public class BatchCompactionUtilsTest extends AbstractCompactionTest { @@ -291,4 +292,141 @@ public void testFollowedBatchChunkWriter2() throws PageException { } Assert.fail(); } + + @Test + public void testMapAlignedChunkMetadata1() { + List valueChunkMetadatas = + Arrays.asList( + new ChunkMetadata("s0", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null), + new ChunkMetadata("s1", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null), + new ChunkMetadata("s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null), + null, + new ChunkMetadata( + "s4", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null)); + AlignedChunkMetadata alignedChunkMetadata = + new AlignedChunkMetadata(new ChunkMetadata(), valueChunkMetadatas); + List measurementSchemas = + Arrays.asList( + new MeasurementSchema("s0", TSDataType.INT32), + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s4", TSDataType.INT32)); + AlignedChunkMetadata newAlignedChunkMetadata = + AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( + alignedChunkMetadata, measurementSchemas); + Assert.assertEquals( + newAlignedChunkMetadata.getValueChunkMetadataList().stream() + .map(IChunkMetadata::getMeasurementUid) + .collect(Collectors.toList()), + Arrays.asList("s0", "s1", "s2", "s4")); + } + + @Test + public void testMapAlignedChunkMetadata2() { + List valueChunkMetadatas = + Arrays.asList( + new ChunkMetadata("s4", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null), + null); + AlignedChunkMetadata alignedChunkMetadata = + new AlignedChunkMetadata(new ChunkMetadata(), valueChunkMetadatas); + List measurementSchemas = + Arrays.asList( + new MeasurementSchema("s0", TSDataType.INT32), + new MeasurementSchema("s4", TSDataType.INT32)); + AlignedChunkMetadata newAlignedChunkMetadata = + AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( + alignedChunkMetadata, measurementSchemas); + Assert.assertEquals( + newAlignedChunkMetadata.getValueChunkMetadataList().stream() + .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) + .collect(Collectors.toList()), + Arrays.asList(null, "s4")); + } + + @Test + public void testMapAlignedChunkMetadata3() { + List valueChunkMetadatas = + Arrays.asList( + new ChunkMetadata("s0", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null), + new ChunkMetadata("s1", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null), + new ChunkMetadata( + "s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null)); + AlignedChunkMetadata alignedChunkMetadata1 = + new AlignedChunkMetadata(new ChunkMetadata(), valueChunkMetadatas); + + valueChunkMetadatas = + Arrays.asList( + new ChunkMetadata("s3", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null), + new ChunkMetadata( + "s4", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null)); + AlignedChunkMetadata alignedChunkMetadata2 = + new AlignedChunkMetadata(new ChunkMetadata(), valueChunkMetadatas); + List measurementSchemas = + Arrays.asList( + new MeasurementSchema("s0", TSDataType.INT32), + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s3", TSDataType.INT32), + new MeasurementSchema("s4", TSDataType.INT32)); + AlignedChunkMetadata newAlignedChunkMetadata1 = + AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( + alignedChunkMetadata1, measurementSchemas); + Assert.assertEquals( + newAlignedChunkMetadata1.getValueChunkMetadataList().stream() + .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) + .collect(Collectors.toList()), + Arrays.asList("s0", "s1", "s2", null, null)); + + AlignedChunkMetadata newAlignedChunkMetadata2 = + AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( + alignedChunkMetadata2, measurementSchemas); + Assert.assertEquals( + newAlignedChunkMetadata2.getValueChunkMetadataList().stream() + .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) + .collect(Collectors.toList()), + Arrays.asList(null, null, null, "s3", "s4")); + } + + @Test + public void testMapAlignedChunkMetadata4() { + List valueChunkMetadatas = + Arrays.asList( + null, + new ChunkMetadata( + "s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null)); + AlignedChunkMetadata alignedChunkMetadata1 = + new AlignedChunkMetadata(new ChunkMetadata(), valueChunkMetadatas); + + valueChunkMetadatas = + Arrays.asList( + null, + null, + null, + null, + new ChunkMetadata( + "s4", TSDataType.INT32, TSEncoding.RLE, CompressionType.LZ4, 0, null)); + AlignedChunkMetadata alignedChunkMetadata2 = + new AlignedChunkMetadata(new ChunkMetadata(), valueChunkMetadatas); + List measurementSchemas = + Arrays.asList( + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s4", TSDataType.INT32)); + AlignedChunkMetadata newAlignedChunkMetadata1 = + AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( + alignedChunkMetadata1, measurementSchemas); + Assert.assertEquals( + newAlignedChunkMetadata1.getValueChunkMetadataList().stream() + .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) + .collect(Collectors.toList()), + Arrays.asList("s2", null)); + + AlignedChunkMetadata newAlignedChunkMetadata2 = + AlignedSeriesBatchCompactionUtils.fillAlignedChunkMetadataBySchemaList( + alignedChunkMetadata2, measurementSchemas); + Assert.assertEquals( + newAlignedChunkMetadata2.getValueChunkMetadataList().stream() + .map(chunkMetadata -> chunkMetadata == null ? null : chunkMetadata.getMeasurementUid()) + .collect(Collectors.toList()), + Arrays.asList(null, "s4")); + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTaskQueueTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTaskQueueTest.java index 2ed0ec5e837d..d6d341acd409 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTaskQueueTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/utils/CompactionTaskQueueTest.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.utils; import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.db.exception.StorageEngineException; import org.apache.iotdb.db.storageengine.dataregion.compaction.AbstractCompactionTest; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.AbstractCompactionTask; @@ -51,19 +52,22 @@ public class CompactionTaskQueueTest extends AbstractCompactionTest { SystemInfo.getInstance().getTotalFileLimitForCompaction(); @Before - public void setup() { + public void setup() + throws IOException, InterruptedException, MetadataException, WriteProcessException { SystemInfo.getInstance().getCompactionMemoryCost().set(0); SystemInfo.getInstance().getCompactionFileNumCost().set(0); SystemInfo.getInstance().setMemorySizeForCompaction(2000); SystemInfo.getInstance().setTotalFileLimitForCompactionTask(50); + super.setUp(); } @After - public void teardown() { + public void teardown() throws StorageEngineException, IOException { SystemInfo.getInstance().getCompactionMemoryCost().set(0); SystemInfo.getInstance().getCompactionFileNumCost().set(0); SystemInfo.getInstance().setMemorySizeForCompaction(originalMemorySizeForCompaction); SystemInfo.getInstance().setTotalFileLimitForCompactionTask(originalFileNumLimitForCompaction); + super.tearDown(); } @Test diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 7b268455b5ce..0227477cfb43 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1043,6 +1043,7 @@ expired_data_ratio=0.3 timestamp_precision=ms # When the timestamp precision check is enabled, the timestamps those are over 13 digits for ms precision, or over 16 digits for us precision are not allowed to be inserted. +# For all precisions, ms, us and ns, the timestamps cannot exceed the range of [-9223372036854775808, 9223372036854775807], regardless of whether the check is enabled or not. # effectiveMode: first_start # Datatype: Boolean timestamp_precision_check_enabled=true @@ -1171,6 +1172,11 @@ enable_unseq_space_compaction=true # Datatype: boolean enable_cross_space_compaction=true +# enable auto repair unsorted file by compaction +# effectiveMode: hot_reload +# Datatype: boolean +enable_auto_repair_compaction=true + # the selector of cross space compaction task # effectiveMode: restart # Options: rewrite @@ -1428,7 +1434,7 @@ delete_wal_files_period_in_ms=20000 # If this value is set smaller than 0, it will default to 50 * 1024 * 1024 * 1024 bytes (50GB). # effectiveMode: hot_reload # Datatype: long -iot_consensus_throttle_threshold_in_byte=53687091200 +wal_throttle_threshold_in_byte=53687091200 # Maximum wait time of write cache in IoTConsensus # If this value is less than or equal to 0, use the default value 10 * 1000 ms (10s) @@ -1837,7 +1843,7 @@ procedure_completed_clean_interval=30 # Default ttl of completed procedure, time unit is second # effectiveMode: restart # Datatype: int -procedure_completed_evict_ttl=800 +procedure_completed_evict_ttl=60 #################### ### MQTT Broker Configuration @@ -1957,4 +1963,4 @@ write_request_remote_dispatch_max_retry_duration_in_ms=60000 # Current unknown errors includes EXECUTE_STATEMENT_ERROR(301) and INTERNAL_SERVER_ERROR(305) # effectiveMode: hot_reload # Datatype: boolean -enable_retry_for_unknown_error=false \ No newline at end of file +enable_retry_for_unknown_error=false diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java index aa49355b31f5..e0a2ea0b635a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/client/request/AsyncRequestManager.java @@ -94,7 +94,7 @@ public final void sendAsyncRequest( sendAsyncRequest(requestContext, 1, null); } - private void sendAsyncRequest( + public void sendAsyncRequest( AsyncRequestContext requestContext, int retryNum, Long timeoutInMs) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java index 3d86dcc663d1..67f3a0265afb 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/plugin/builtin/BuiltinPipePlugin.java @@ -139,8 +139,10 @@ public String getClassName() { CHANGING_VALUE_SAMPLING_PROCESSOR.getPipePluginName().toUpperCase(), THROWING_EXCEPTION_PROCESSOR.getPipePluginName().toUpperCase(), AGGREGATE_PROCESSOR.getPipePluginName().toUpperCase(), + COUNT_POINT_PROCESSOR.getPipePluginName().toUpperCase(), STANDARD_STATISTICS_PROCESSOR.getPipePluginName().toUpperCase(), TUMBLING_WINDOWING_PROCESSOR.getPipePluginName().toUpperCase(), + PIPE_CONSENSUS_PROCESSOR.getPipePluginName().toUpperCase(), // Connectors DO_NOTHING_CONNECTOR.getPipePluginName().toUpperCase(), IOTDB_THRIFT_CONNECTOR.getPipePluginName().toUpperCase(), @@ -152,6 +154,7 @@ public String getClassName() { WEBSOCKET_CONNECTOR.getPipePluginName().toUpperCase(), OPC_UA_CONNECTOR.getPipePluginName().toUpperCase(), WRITE_BACK_CONNECTOR.getPipePluginName().toUpperCase(), + PIPE_CONSENSUS_ASYNC_CONNECTOR.getPipePluginName().toUpperCase(), // Sinks IOTDB_THRIFT_SYNC_SINK.getPipePluginName().toUpperCase(), IOTDB_THRIFT_ASYNC_SINK.getPipePluginName().toUpperCase(), @@ -159,5 +162,6 @@ public String getClassName() { WEBSOCKET_SINK.getPipePluginName().toUpperCase(), OPC_UA_SINK.getPipePluginName().toUpperCase(), WRITE_BACK_SINK.getPipePluginName().toUpperCase(), - SUBSCRIPTION_SINK.getPipePluginName().toUpperCase()))); + SUBSCRIPTION_SINK.getPipePluginName().toUpperCase(), + PIPE_CONSENSUS_ASYNC_SINK.getPipePluginName().toUpperCase()))); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java index 79086799fcd5..58b8c06192cf 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java @@ -24,6 +24,12 @@ public class PipeExtractorConstant { public static final String EXTRACTOR_KEY = "extractor"; public static final String SOURCE_KEY = "source"; + public static final String EXTRACTOR_DIALECT_KEY = "extractor.dialect"; + public static final String SOURCE_DIALECT_KEY = "source.dialect"; + public static final String EXTRACTOR_DIALECT_TREE_VALUE = "tree"; + public static final String EXTRACTOR_DIALECT_TABLE_VALUE = "table"; + public static final String EXTRACTOR_DIALECT_DEFAULT_VALUE = EXTRACTOR_DIALECT_TREE_VALUE; + public static final String EXTRACTOR_INCLUSION_KEY = "extractor.inclusion"; public static final String SOURCE_INCLUSION_KEY = "source.inclusion"; public static final String EXTRACTOR_INCLUSION_DEFAULT_VALUE = "data.insert"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java index a14cfe3e214f..6a4dd9e90679 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBSslSyncConnector.java @@ -65,7 +65,7 @@ public abstract class IoTDBSslSyncConnector extends IoTDBConnector { protected IoTDBSyncClientManager clientManager; @Override - public void validate(PipeParameterValidator validator) throws Exception { + public void validate(final PipeParameterValidator validator) throws Exception { super.validate(validator); final PipeParameters parameters = validator.getParameters(); @@ -90,7 +90,8 @@ public void validate(PipeParameterValidator validator) throws Exception { } @Override - public void customize(PipeParameters parameters, PipeConnectorRuntimeConfiguration configuration) + public void customize( + final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) throws Exception { super.customize(parameters, configuration); @@ -127,14 +128,14 @@ public void customize(PipeParameters parameters, PipeConnectorRuntimeConfigurati } protected abstract IoTDBSyncClientManager constructClient( - List nodeUrls, - boolean useSSL, - String trustStorePath, - String trustStorePwd, - boolean useLeaderCache, - String loadBalanceStrategy, - boolean shouldReceiverConvertOnTypeMismatch, - String loadTsFileStrategy); + final List nodeUrls, + final boolean useSSL, + final String trustStorePath, + final String trustStorePwd, + final boolean useLeaderCache, + final String loadBalanceStrategy, + final boolean shouldReceiverConvertOnTypeMismatch, + final String loadTsFileStrategy); @Override public void handshake() throws Exception { @@ -145,7 +146,7 @@ public void handshake() throws Exception { public void heartbeat() { try { handshake(); - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn( "Failed to reconnect to target server, because: {}. Try to reconnect later.", e.getMessage(), @@ -190,7 +191,7 @@ protected void transferFilePieces( resp = PipeTransferFilePieceResp.fromTPipeTransferResp( clientAndStatus.getLeft().pipeTransfer(req)); - } catch (Exception e) { + } catch (final Exception e) { clientAndStatus.setRight(false); throw new PipeConnectionException( String.format( @@ -228,10 +229,10 @@ protected void transferFilePieces( } protected abstract PipeTransferFilePieceReq getTransferSingleFilePieceReq( - String fileName, long position, byte[] payLoad) throws IOException; + final String fileName, final long position, final byte[] payLoad) throws IOException; protected abstract PipeTransferFilePieceReq getTransferMultiFilePieceReq( - String fileName, long position, byte[] payLoad) throws IOException; + final String fileName, final long position, final byte[] payLoad) throws IOException; @Override public void close() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java index 4c96f0ec29df..0c1ffb56224a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/options/PipeInclusionOptions.java @@ -91,7 +91,7 @@ public class PipeInclusionOptions { Arrays.asList( "data.delete", "schema.database.drop", - "schema.timeseries.ordinary.drop", + "schema.timeseries.ordinary.delete", "schema.timeseries.view.drop", "schema.timeseries.template.drop", "schema.timeseries.template.unset", diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 5d7cc60ab2fc..84e7c05ed401 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -189,10 +189,10 @@ public byte[] serialize() { return stream.toByteArray(); } - public void serialize(OutputStream stream) throws IOException { + public void serialize(final OutputStream stream) throws IOException { ReadWriteIOUtils.write(tableName, stream); ReadWriteIOUtils.write(columnSchemaMap.size(), stream); - for (TsTableColumnSchema columnSchema : columnSchemaMap.values()) { + for (final TsTableColumnSchema columnSchema : columnSchemaMap.values()) { TsTableColumnSchemaUtil.serialize(columnSchema, stream); } ReadWriteIOUtils.write(props, stream); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCUtil.java index ac9fa72d8e4d..651eb50b21a2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTableInternalRPCUtil.java @@ -30,35 +30,36 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; public class TsTableInternalRPCUtil { private TsTableInternalRPCUtil() { - // do nothing + // Do nothing } - public static byte[] serializeBatchTsTable(Map> tableMap) { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + public static byte[] serializeBatchTsTable(final Map> tableMap) { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try { ReadWriteIOUtils.write(tableMap.size(), outputStream); - for (Map.Entry> entry : tableMap.entrySet()) { + for (final Map.Entry> entry : tableMap.entrySet()) { ReadWriteIOUtils.write(entry.getKey(), outputStream); ReadWriteIOUtils.write(entry.getValue().size(), outputStream); - for (TsTable table : entry.getValue()) { + for (final TsTable table : entry.getValue()) { table.serialize(outputStream); } } - } catch (IOException ignored) { - // won't happen + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException } return outputStream.toByteArray(); } - public static Map> deserializeBatchTsTable(byte[] bytes) { - InputStream inputStream = new ByteArrayInputStream(bytes); - Map> result = new HashMap<>(); + public static Map> deserializeBatchTsTable(final byte[] bytes) { + final InputStream inputStream = new ByteArrayInputStream(bytes); + final Map> result = new HashMap<>(); try { - int dbNum = ReadWriteIOUtils.readInt(inputStream); + final int dbNum = ReadWriteIOUtils.readInt(inputStream); String database; int tableNum; List tableList; @@ -71,8 +72,8 @@ public static Map> deserializeBatchTsTable(byte[] bytes) { } result.put(database, tableList); } - } catch (IOException ignored) { - // won't happen + } catch (final IOException ignored) { + // ByteArrayInputStream won't throw IOException } return result; } @@ -89,29 +90,76 @@ public static byte[] serializeSingleTsTable(final String database, final TsTable } public static Pair deserializeSingleTsTable(final byte[] bytes) { - InputStream inputStream = new ByteArrayInputStream(bytes); + final InputStream inputStream = new ByteArrayInputStream(bytes); try { - String database = ReadWriteIOUtils.readString(inputStream); - TsTable table = TsTable.deserialize(inputStream); - return new Pair<>(database, table); - } catch (IOException ignored) { + return new Pair<>(ReadWriteIOUtils.readString(inputStream), TsTable.deserialize(inputStream)); + } catch (final IOException ignored) { // ByteArrayInputStream won't throw IOException } throw new IllegalStateException(); } public static byte[] serializeTableInitializationInfo( - Map> usingTableMap, Map> preCreateTableMap) { - byte[] usingBytes = serializeBatchTsTable(usingTableMap); - byte[] preCreateBytes = serializeBatchTsTable(preCreateTableMap); - byte[] result = new byte[usingBytes.length + preCreateBytes.length]; + final Map> usingTableMap, + final Map> preCreateTableMap) { + final byte[] usingBytes = serializeBatchTsTable(usingTableMap); + final byte[] preCreateBytes = serializeBatchTsTable(preCreateTableMap); + final byte[] result = new byte[usingBytes.length + preCreateBytes.length]; System.arraycopy(usingBytes, 0, result, 0, usingBytes.length); System.arraycopy(preCreateBytes, 0, result, usingBytes.length, preCreateBytes.length); return result; } public static Pair>, Map>> - deserializeTableInitializationInfo(byte[] bytes) { + deserializeTableInitializationInfo(final byte[] bytes) { return new Pair<>(deserializeBatchTsTable(bytes), deserializeBatchTsTable(bytes)); } + + public static byte[] serializeTableFetchResult( + final Map> fetchTableMap) { + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(fetchTableMap.size(), outputStream); + for (final Map.Entry> entry : fetchTableMap.entrySet()) { + ReadWriteIOUtils.write(entry.getKey(), outputStream); + ReadWriteIOUtils.write(entry.getValue().size(), outputStream); + for (final Map.Entry tableEntry : entry.getValue().entrySet()) { + ReadWriteIOUtils.write(tableEntry.getKey(), outputStream); + ReadWriteIOUtils.write(Objects.nonNull(tableEntry.getValue()), outputStream); + if (Objects.nonNull(tableEntry.getValue())) { + tableEntry.getValue().serialize(outputStream); + } + } + } + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + return outputStream.toByteArray(); + } + + public static Map> deserializeTsTableFetchResult( + final byte[] bytes) { + final InputStream inputStream = new ByteArrayInputStream(bytes); + final Map> result = new HashMap<>(); + try { + int dbNum = ReadWriteIOUtils.readInt(inputStream); + String database; + int tableNum; + Map tableMap; + for (int i = 0; i < dbNum; i++) { + database = ReadWriteIOUtils.readString(inputStream); + tableNum = ReadWriteIOUtils.readInt(inputStream); + tableMap = new HashMap<>(tableNum); + for (int j = 0; j < tableNum; j++) { + tableMap.put( + ReadWriteIOUtils.readString(inputStream), + ReadWriteIOUtils.readBool(inputStream) ? TsTable.deserialize(inputStream) : null); + } + result.put(database, tableMap); + } + } catch (final IOException ignored) { + // ByteArrayInputStream won't throw IOException + } + return result; + } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/PathUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/PathUtils.java index 089fef841e2e..005003d9b250 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/PathUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/utils/PathUtils.java @@ -213,4 +213,11 @@ public static String qualifyDatabaseName(String databaseName) { } return databaseName; } + + public static String unQualifyDatabaseName(String databaseName) { + if (databaseName != null && databaseName.startsWith("root.")) { + databaseName = databaseName.substring(5); + } + return databaseName; + } } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index bfcdaf96fb16..804037da3d5a 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -73,6 +73,16 @@ statement // Load Statement | loadTsFileStatement + // Pipe Statement + | createPipeStatement + | alterPipeStatement + | dropPipeStatement + | startPipeStatement + | stopPipeStatement + | showPipesStatement + | createPipePluginStatement + | dropPipePluginStatement + | showPipePluginsStatement // Show Statement | showDevicesStatement @@ -83,6 +93,7 @@ statement | showRegionsStatement | showDataNodesStatement | showConfigNodesStatement + | showAINodesStatement | showClusterIdStatement | showRegionIdStatement | showTimeSlotListStatement @@ -233,6 +244,110 @@ loadTsFileStatement ; +// -------------------------------------------- Pipe Statement --------------------------------------------------------- +createPipeStatement + : CREATE PIPE (IF NOT EXISTS)? pipeName=identifier + ((extractorAttributesClause? processorAttributesClause? connectorAttributesClause) + | connectorAttributesWithoutWithSinkClause) + ; + +extractorAttributesClause + : WITH (EXTRACTOR | SOURCE) + '(' + (extractorAttributeClause ',')* extractorAttributeClause? + ')' + ; + +extractorAttributeClause + : extractorKey=string EQ extractorValue=string + ; + +processorAttributesClause + : WITH PROCESSOR + '(' + (processorAttributeClause ',')* processorAttributeClause? + ')' + ; + +processorAttributeClause + : processorKey=string EQ processorValue=string + ; + +connectorAttributesClause + : WITH (CONNECTOR | SINK) + '(' + (connectorAttributeClause ',')* connectorAttributeClause? + ')' + ; + +connectorAttributesWithoutWithSinkClause + : '(' + (connectorAttributeClause ',')* connectorAttributeClause? + ')' + ; + +connectorAttributeClause + : connectorKey=string EQ connectorValue=string + ; + +alterPipeStatement + : ALTER PIPE (IF EXISTS)? pipeName=identifier + alterExtractorAttributesClause? + alterProcessorAttributesClause? + alterConnectorAttributesClause? + ; + +alterExtractorAttributesClause + : (MODIFY | REPLACE) (EXTRACTOR | SOURCE) + '(' + (extractorAttributeClause ',')* extractorAttributeClause? + ')' + ; + +alterProcessorAttributesClause + : (MODIFY | REPLACE) PROCESSOR + '(' + (processorAttributeClause ',')* processorAttributeClause? + ')' + ; + +alterConnectorAttributesClause + : (MODIFY | REPLACE) (CONNECTOR | SINK) + '(' + (connectorAttributeClause ',')* connectorAttributeClause? + ')' + ; + +dropPipeStatement + : DROP PIPE (IF EXISTS)? pipeName=identifier + ; + +startPipeStatement + : START PIPE pipeName=identifier + ; + +stopPipeStatement + : STOP PIPE pipeName=identifier + ; + +showPipesStatement + : SHOW ((PIPE pipeName=identifier) | PIPES (WHERE (CONNECTOR | SINK) USED BY pipeName=identifier)?) + ; + +createPipePluginStatement + : CREATE PIPEPLUGIN (IF NOT EXISTS)? pluginName=identifier AS className=string uriClause + ; + +dropPipePluginStatement + : DROP PIPEPLUGIN (IF EXISTS)? pluginName=identifier + ; + +showPipePluginsStatement + : SHOW PIPEPLUGINS + ; + + + // -------------------------------------------- Show Statement --------------------------------------------------------- showDevicesStatement : SHOW DEVICES FROM tableName=qualifiedName @@ -266,6 +381,10 @@ showConfigNodesStatement : SHOW CONFIGNODES ; +showAINodesStatement + : SHOW AINODES + ; + showClusterIdStatement : SHOW CLUSTERID ; @@ -713,9 +832,9 @@ nonReserved // IMPORTANT: this rule must only contain tokens. Nested rules are not supported. See SqlParser.exitNonReserved : ABSENT | ADD | ADMIN | AFTER | ALL | ANALYZE | ANY | ARRAY | ASC | AT | ATTRIBUTE | AUTHORIZATION | BEGIN | BERNOULLI | BOTH - | CACHE | CALL | CALLED | CASCADE | CATALOG | CATALOGS | CHAR | CHARACTER | CHARSET | CLEAR | CLUSTER | CLUSTERID | COLUMN | COLUMNS | COMMENT | COMMIT | COMMITTED | CONDITION | CONDITIONAL | CONFIGNODES | CONFIGURATION | COPARTITION | COUNT | CURRENT + | CACHE | CALL | CALLED | CASCADE | CATALOG | CATALOGS | CHAR | CHARACTER | CHARSET | CLEAR | CLUSTER | CLUSTERID | COLUMN | COLUMNS | COMMENT | COMMIT | COMMITTED | CONDITION | CONDITIONAL | CONFIGNODES | CONFIGURATION | CONNECTOR | COPARTITION | COUNT | CURRENT | DATA | DATABASE | DATABASES | DATANODES | DATE | DAY | DECLARE | DEFAULT | DEFINE | DEFINER | DENY | DESC | DESCRIPTOR | DETAILS| DETERMINISTIC | DEVICES | DISTRIBUTED | DO | DOUBLE - | ELSEIF | EMPTY | ENCODING | ERROR | EXCLUDING | EXPLAIN + | ELSEIF | EMPTY | ENCODING | ERROR | EXCLUDING | EXPLAIN | EXTRACTOR | FETCH | FILL | FILTER | FINAL | FIRST | FLUSH | FOLLOWING | FORMAT | FUNCTION | FUNCTIONS | GRACE | GRANT | GRANTED | GRANTS | GRAPHVIZ | GROUPS | HOUR @@ -723,16 +842,16 @@ nonReserved | JSON | KEEP | KEY | KEYS | KILL | LANGUAGE | LAST | LATERAL | LEADING | LEAVE | LEVEL | LIMIT | LINEAR | LOAD | LOCAL | LOGICAL | LOOP - | MAP | MATCH | MATCHED | MATCHES | MATCH_RECOGNIZE | MATERIALIZED | MEASUREMENT | MEASURES | MERGE | MICROSECOND | MIGRATE | MILLISECOND | MINUTE | MONTH + | MAP | MATCH | MATCHED | MATCHES | MATCH_RECOGNIZE | MATERIALIZED | MEASUREMENT | MEASURES | MERGE | MICROSECOND | MIGRATE | MILLISECOND | MINUTE | MODIFY | MONTH | NANOSECOND | NESTED | NEXT | NFC | NFD | NFKC | NFKD | NO | NODEID | NONE | NULLIF | NULLS | OBJECT | OF | OFFSET | OMIT | ONE | ONLY | OPTION | ORDINALITY | OUTPUT | OVER | OVERFLOW - | PARTITION | PARTITIONS | PASSING | PAST | PATH | PATTERN | PER | PERIOD | PERMUTE | PLAN | POSITION | PRECEDING | PRECISION | PRIVILEGES | PREVIOUS | PROCESSLIST | PROPERTIES | PRUNE + | PARTITION | PARTITIONS | PASSING | PAST | PATH | PATTERN | PER | PERIOD | PERMUTE | PIPE | PIPEPLUGIN | PIPEPLUGINS | PIPES | PLAN | POSITION | PRECEDING | PRECISION | PRIVILEGES | PREVIOUS | PROCESSLIST | PROCESSOR | PROPERTIES | PRUNE | QUERIES | QUERY | QUOTES | RANGE | READ | READONLY | REFRESH | REGION | REGIONID | REGIONS | RENAME | REPAIR | REPEAT | REPEATABLE | REPLACE | RESET | RESPECT | RESTRICT | RETURN | RETURNING | RETURNS | REVOKE | ROLE | ROLES | ROLLBACK | ROW | ROWS | RUNNING | SERIESSLOTID | SCALAR | SCHEMA | SCHEMAS | SECOND | SECURITY | SEEK | SERIALIZABLE | SESSION | SET | SETS - | SHOW | SOME | START | STATS | SUBSET | SUBSTRING | SYSTEM + | SHOW | SINK | SOME | SOURCE | START | STATS | STOP | SUBSET | SUBSTRING | SYSTEM | TABLES | TABLESAMPLE | TEXT | TEXT_STRING | TIES | TIME | TIMEPARTITION | TIMESERIES | TIMESLOTID | TIMESTAMP | TO | TRAILING | TRANSACTION | TRUNCATE | TRY_CAST | TYPE - | UNBOUNDED | UNCOMMITTED | UNCONDITIONAL | UNIQUE | UNKNOWN | UNMATCHED | UNTIL | UPDATE | URI | USE | USER | UTF16 | UTF32 | UTF8 + | UNBOUNDED | UNCOMMITTED | UNCONDITIONAL | UNIQUE | UNKNOWN | UNMATCHED | UNTIL | UPDATE | URI | USE | USED | USER | UTF16 | UTF32 | UTF8 | VALIDATE | VALUE | VARIABLES | VARIATION | VERBOSE | VERSION | VIEW | WEEK | WHILE | WINDOW | WITHIN | WITHOUT | WORK | WRAPPER | WRITE | YEAR @@ -743,6 +862,7 @@ ABSENT: 'ABSENT'; ADD: 'ADD'; ADMIN: 'ADMIN'; AFTER: 'AFTER'; +AINODES: 'AINODES'; ALL: 'ALL'; ALTER: 'ALTER'; ANALYZE: 'ANALYZE'; @@ -782,6 +902,7 @@ CONDITION: 'CONDITION'; CONDITIONAL: 'CONDITIONAL'; CONFIGNODES: 'CONFIGNODES'; CONFIGURATION: 'CONFIGURATION'; +CONNECTOR: 'CONNECTOR'; CONSTRAINT: 'CONSTRAINT'; COUNT: 'COUNT'; COPARTITION: 'COPARTITION'; @@ -836,6 +957,7 @@ EXECUTE: 'EXECUTE'; EXISTS: 'EXISTS'; EXPLAIN: 'EXPLAIN'; EXTRACT: 'EXTRACT'; +EXTRACTOR: 'EXTRACTOR'; FALSE: 'FALSE'; FETCH: 'FETCH'; FILL: 'FILL'; @@ -922,6 +1044,7 @@ MICROSECOND: 'US'; MIGRATE: 'MIGRATE'; MILLISECOND: 'MS'; MINUTE: 'MINUTE' | 'M'; +MODIFY: 'MODIFY'; MONTH: 'MONTH' | 'MO'; NANOSECOND: 'NS'; NATURAL: 'NATURAL'; @@ -964,6 +1087,10 @@ PATTERN: 'PATTERN'; PER: 'PER'; PERIOD: 'PERIOD'; PERMUTE: 'PERMUTE'; +PIPE: 'PIPE'; +PIPEPLUGIN: 'PIPEPLUGIN'; +PIPEPLUGINS: 'PIPEPLUGINS'; +PIPES: 'PIPES'; PLAN : 'PLAN'; POSITION: 'POSITION'; PRECEDING: 'PRECEDING'; @@ -972,6 +1099,7 @@ PREPARE: 'PREPARE'; PRIVILEGES: 'PRIVILEGES'; PREVIOUS: 'PREVIOUS'; PROCESSLIST: 'PROCESSLIST'; +PROCESSOR: 'PROCESSOR'; PROPERTIES: 'PROPERTIES'; PRUNE: 'PRUNE'; QUERIES: 'QUERIES'; @@ -1018,10 +1146,13 @@ SESSION: 'SESSION'; SET: 'SET'; SETS: 'SETS'; SHOW: 'SHOW'; +SINK: 'SINK'; SKIP_TOKEN: 'SKIP'; SOME: 'SOME'; +SOURCE: 'SOURCE'; START: 'START'; STATS: 'STATS'; +STOP: 'STOP'; SUBSET: 'SUBSET'; SUBSTRING: 'SUBSTRING'; SYSTEM: 'SYSTEM'; @@ -1058,6 +1189,7 @@ UNTIL: 'UNTIL'; UPDATE: 'UPDATE'; URI: 'URI'; USE: 'USE'; +USED: 'USED'; USER: 'USER'; USING: 'USING'; UTF16: 'UTF16'; diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 05e995af8f9f..5c1ad65b4fbd 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1044,6 +1044,11 @@ struct TShowTableResp { 2: optional list tableInfoList } +struct TFetchTableResp { + 1: required common.TSStatus status + 2: optional binary tableInfoMap +} + struct TTableInfo { 1: required string tableName // TTL is stored as string in table props @@ -1785,5 +1790,7 @@ service IConfigNodeRPCService { common.TSStatus alterTable(TAlterTableReq req) TShowTableResp showTables(string database) + + TFetchTableResp fetchTables(map> fetchTableMap) } diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index a1794537cef1..e2fb51834dcc 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -1057,17 +1057,4 @@ service MPPDataExchangeService { /** Empty rpc, only for connection test */ common.TSStatus testConnectionEmptyRPC() -} - -service IAINodeInternalRPCService{ - /** - * Fecth the data of the specified time series - */ - TFetchTimeseriesResp fetchTimeseries(TFetchTimeseriesReq req) - - /** - * Fetch rest data for a specified fetchTimeseries - */ - TFetchMoreDataResp fetchMoreData(TFetchMoreDataReq req) - -} +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index cf45ff69e1a1..ef75293b028b 100644 --- a/pom.xml +++ b/pom.xml @@ -135,7 +135,7 @@ is for ensuring the SNAPSHOT will stay available. We should however have the Ratis folks do a new release soon, as releasing with this version is more than sub-ideal. --> - 3.1.1-0133c90-SNAPSHOT + 3.1.1 1.0.4 1.1.20 3.5.18 @@ -166,7 +166,7 @@ 0.14.1 1.9 1.5.6-3 - 1.2.0-c3437a41-SNAPSHOT + 1.2.0-240924-SNAPSHOT + + enable-sbom-check + + + + + org.cyclonedx + cyclonedx-maven-plugin + + false + + apache-${project.artifactId}-${project.version}-sbom + + + + package + + makeAggregateBom + + + + + + org.codehaus.mojo + xml-maven-plugin + 1.1.0 + + false + + + package + + transform + + + + +

${project.basedir}/target/ + apache-${project.artifactId}-${project.version}-sbom.xml + src/main/xslt/sbom-filter.xsl + ${project.basedir}/target/ + + + transformed.json + + + + + + + + + + net.sf.saxon + Saxon-HE + 12.5 + + + + + org.codehaus.gmaven + groovy-maven-plugin + 2.1.1 + + false + + + compare-with-reference-list + verify + + execute + + + src/main/groovy/checkDependencies.groovy + + + + + + org.apache.groovy + groovy + 4.0.22 + + + + + + apache-release diff --git a/src/main/groovy/checkDependencies.groovy b/src/main/groovy/checkDependencies.groovy new file mode 100644 index 000000000000..8df42473d349 --- /dev/null +++ b/src/main/groovy/checkDependencies.groovy @@ -0,0 +1,60 @@ +package src.main.groovy +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import groovy.json.JsonSlurper + +if(Boolean.parseBoolean(properties['skipDependencyCheck']).booleanValue()) { + println "Skipping dependency check" + return +} + +def jsonSlurper = new JsonSlurper() + +var referenceFile = new File(basedir, "dependencies.json") +if(!referenceFile.exists()) { + throw new RuntimeException("Missing Reference: dependencies.json") +} +def referenceJson = jsonSlurper.parse(referenceFile) + +var curBuildFile = new File(project.build.directory, "apache-${project.artifactId}-${project.version}-sbom.transformed.json") +if(!curBuildFile.exists()) { + throw new RuntimeException("Missing Build: apache-${project.artifactId}-${project.version}-sbom.transformed.json") +} +def curBuildJson = jsonSlurper.parse(curBuildFile) + +def differencesFound = false +referenceJson.dependencies.each { + if(!curBuildJson.dependencies.contains(it)) { + println "current build has removed a dependency: " + it + differencesFound = true + } +} +curBuildJson.dependencies.each { + if(!referenceJson.dependencies.contains(it)) { + println "current build has added a dependency: " + it + differencesFound = true + } +} + +if(differencesFound) { + println "Differences were found between the information in ${referenceFile.getPath()} and ${curBuildFile.toPath()}" + println "The simplest fix for this, is to replace the content of ${referenceFile.getPath()} with that of ${curBuildFile.toPath()} and to inspect the diff of the resulting file in your IDE of choice." + throw new RuntimeException("Differences found.") +} \ No newline at end of file diff --git a/src/main/xslt/sbom-filter.xsl b/src/main/xslt/sbom-filter.xsl new file mode 100644 index 000000000000..3450837c23a9 --- /dev/null +++ b/src/main/xslt/sbom-filter.xsl @@ -0,0 +1,41 @@ + + + + + + + { + "dependencies": [ + + + ":", + + ] +} + + \ No newline at end of file