diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT.java similarity index 98% rename from integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT.java rename to integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT.java index c608bf3e24c0..aa568b583c1b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT.java @@ -17,18 +17,17 @@ * under the License. */ -package org.apache.iotdb.db.it.regionscan; +package org.apache.iotdb.db.it.schema.regionscan; import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.util.AbstractSchemaIT; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; import java.sql.Connection; import java.sql.ResultSet; @@ -45,9 +44,8 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; -@RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) -public class IoTDBActiveRegionScanIT { +public class IoTDBActiveRegionScanIT extends AbstractSchemaIT { // Data can be viewed in // https://docs.google.com/spreadsheets/d/11tNRIaHmNdFWc0RC4yAloO6JJbmo9S5qxwAqWxeVY6c/edit#gid=0 public static final String[] common_insert_sqls = @@ -145,6 +143,10 @@ public class IoTDBActiveRegionScanIT { public static final String COUNT_TIMESERIES_COLUMN_NAMES = "count(timeseries)"; public static final String COUNT_DEVICES_COLUMN_NAMES = "count(devices)"; + public IoTDBActiveRegionScanIT(SchemaTestMode schemaTestMode) { + super(schemaTestMode); + } + public static void insertData() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT2.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT2.java similarity index 91% rename from integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT2.java rename to integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT2.java index be4751495de4..da56d3078b01 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanIT2.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanIT2.java @@ -17,21 +17,22 @@ * under the License. */ -package org.apache.iotdb.db.it.regionscan; +package org.apache.iotdb.db.it.schema.regionscan; import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -@RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBActiveRegionScanIT2 extends IoTDBActiveRegionScanIT { + public IoTDBActiveRegionScanIT2(SchemaTestMode schemaTestMode) { + super(schemaTestMode); + } + @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv() diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT.java similarity index 85% rename from integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT.java rename to integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT.java index d977a6d0132e..0c10f507aa21 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT.java @@ -17,34 +17,32 @@ * under the License. */ -package org.apache.iotdb.db.it.regionscan; +package org.apache.iotdb.db.it.schema.regionscan; import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.util.AbstractSchemaIT; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; import java.sql.Connection; import java.sql.Statement; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.COUNT_DEVICES_COLUMN_NAMES; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.COUNT_TIMESERIES_COLUMN_NAMES; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.SHOW_DEVICES_COLUMN_NAMES; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.SHOW_TIMESERIES_COLUMN_NAMES; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.basicCountActiveDeviceTest; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.basicShowActiveDeviceTest; -import static org.apache.iotdb.db.it.regionscan.IoTDBActiveRegionScanIT.common_insert_sqls; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.COUNT_DEVICES_COLUMN_NAMES; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.COUNT_TIMESERIES_COLUMN_NAMES; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.SHOW_DEVICES_COLUMN_NAMES; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.SHOW_TIMESERIES_COLUMN_NAMES; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.basicCountActiveDeviceTest; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.basicShowActiveDeviceTest; +import static org.apache.iotdb.db.it.schema.regionscan.IoTDBActiveRegionScanIT.common_insert_sqls; import static org.junit.Assert.fail; -@RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) -public class IoTDBActiveRegionScanWithModsIT { +public class IoTDBActiveRegionScanWithModsIT extends AbstractSchemaIT { private static String[] delete_sqls = new String[] { @@ -57,6 +55,10 @@ public class IoTDBActiveRegionScanWithModsIT { "delete from root.sg.unaligned.d2.* where time > 40 and time <= 43" }; + public IoTDBActiveRegionScanWithModsIT(SchemaTestMode schemaTestMode) { + super(schemaTestMode); + } + public static void insertData() { try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT2.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT2.java similarity index 91% rename from integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT2.java rename to integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT2.java index a87240341e01..5adccd1fd698 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/regionscan/IoTDBActiveRegionScanWithModsIT2.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveRegionScanWithModsIT2.java @@ -17,21 +17,22 @@ * under the License. */ -package org.apache.iotdb.db.it.regionscan; +package org.apache.iotdb.db.it.schema.regionscan; import org.apache.iotdb.it.env.EnvFactory; -import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.ClusterIT; import org.apache.iotdb.itbase.category.LocalStandaloneIT; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.experimental.categories.Category; -import org.junit.runner.RunWith; -@RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) public class IoTDBActiveRegionScanWithModsIT2 extends IoTDBActiveRegionScanWithModsIT { + public IoTDBActiveRegionScanWithModsIT2(SchemaTestMode schemaTestMode) { + super(schemaTestMode); + } + @BeforeClass public static void setUp() throws Exception { EnvFactory.getEnv() diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java new file mode 100644 index 000000000000..853ef3c87ae0 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/schema/regionscan/IoTDBActiveSchemaQueryIT.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.schema.regionscan; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.util.AbstractSchemaIT; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runners.Parameterized; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.Statement; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBActiveSchemaQueryIT extends AbstractSchemaIT { + + private static String[] sqls = + new String[] { + "create aligned timeseries root.sg.d1(s1(alias1) int32 tags('tag1'='v1', 'tag2'='v2'), s2 double attributes('attr3'='v3'))", + "CREATE TIMESERIES root.sg.d0.s0 WITH DATATYPE=INT32", + "CREATE TIMESERIES root.sg.d0.s1 WITH DATATYPE=INT32,ENCODING=PLAIN,LOSS=SDT,COMPDEV=2", + "CREATE TIMESERIES root.sg.d0.s2 WITH DATATYPE=INT32,ENCODING=PLAIN,LOSS=SDT, COMPDEV=0.01, COMPMINTIME=2, COMPMAXTIME=15", + "CREATE DEVICE TEMPLATE t1 (s1 INT64 encoding=RLE compressor=SNAPPY, s2 INT32)", + "CREATE DEVICE TEMPLATE t2 aligned(s1 INT64 encoding=RLE compressor=SNAPPY, s2 INT32)", + "SET DEVICE TEMPLATE t1 to root.sg.d2", + "SET DEVICE TEMPLATE t2 to root.sg.d3", + "CREATE TIMESERIES OF DEVICE TEMPLATE ON root.sg.d2", + "CREATE TIMESERIES OF DEVICE TEMPLATE ON root.sg.d3", + "set ttl to root.sg.d3 60000", + }; + + public IoTDBActiveSchemaQueryIT(SchemaTestMode schemaTestMode) { + super(schemaTestMode); + } + + @Parameterized.BeforeParam + public static void before() throws Exception { + SchemaTestMode schemaTestMode = setUpEnvironment(); + if (schemaTestMode.equals(SchemaTestMode.PBTree)) { + allocateMemoryForSchemaRegion(10000); + } + EnvFactory.getEnv().initClusterEnvironment(); + } + + @Parameterized.AfterParam + public static void after() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + tearDownEnvironment(); + } + + @Before + public void setUp() throws Exception { + createSchema(); + } + + @After + public void tearDown() throws Exception { + clearSchema(); + } + + private void createSchema() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + for (String sql : sqls) { + statement.execute(sql); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Test + public void testShowTimeSeries() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // 0. Check environment + Set expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1.s1,alias1,root.sg,INT32,TS_2DIFF,LZ4,{\"tag1\":\"v1\",\"tag2\":\"v2\"},null,null,null,BASE,", + "root.sg.d1.s2,null,root.sg,DOUBLE,GORILLA,LZ4,null,{\"attr3\":\"v3\"},null,null,BASE,", + "root.sg.d0.s0,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d0.s1,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=2},BASE,", + "root.sg.d0.s2,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=0.01, compmintime=2, compmaxtime=15},BASE,", + "root.sg.d2.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d2.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d3.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d3.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,")); + checkResultSet(statement, "show timeseries root.sg.**", expected); + // 1. Without any data, result set should be empty + try (ResultSet resultSet = + statement.executeQuery("show timeseries root.sg.** where time>0")) { + Assert.assertFalse(resultSet.next()); + } + // 2. Insert data and check again. + // - root.db.d0.* time=1 + // - root.db.d1.* time=2 + // - root.db.d2.* time=3 + // - root.db.d3.* time=now() + statement.execute("insert into root.sg.d0(timestamp,s0, s1, s2) values(1,1,1,1)"); + statement.execute("insert into root.sg.d1(timestamp,s1,s2) values(2,2,2)"); + statement.execute("insert into root.sg.d2(timestamp,s1,s2) values(3,3,3)"); + statement.execute("insert into root.sg.d3(timestamp,s1,s2) values(now(),4,4)"); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1.s1,alias1,root.sg,INT32,TS_2DIFF,LZ4,{\"tag1\":\"v1\",\"tag2\":\"v2\"},null,null,null,BASE,", + "root.sg.d1.s2,null,root.sg,DOUBLE,GORILLA,LZ4,null,{\"attr3\":\"v3\"},null,null,BASE,", + "root.sg.d0.s0,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d0.s1,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=2},BASE,", + "root.sg.d0.s2,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=0.01, compmintime=2, compmaxtime=15},BASE,", + "root.sg.d2.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d2.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d3.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d3.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,")); + checkResultSet(statement, "show timeseries root.sg.** where time>0", expected); + checkResultSet( + statement, + "count timeseries root.sg.** where time>0", + new HashSet<>(Collections.singletonList("9,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1.s1,alias1,root.sg,INT32,TS_2DIFF,LZ4,{\"tag1\":\"v1\",\"tag2\":\"v2\"},null,null,null,BASE,", + "root.sg.d1.s2,null,root.sg,DOUBLE,GORILLA,LZ4,null,{\"attr3\":\"v3\"},null,null,BASE,", + "root.sg.d2.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d2.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d3.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d3.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,")); + checkResultSet(statement, "show timeseries root.sg.** where time>1", expected); + checkResultSet( + statement, + "count timeseries root.sg.** where time>1", + new HashSet<>(Collections.singletonList("6,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1.s1,alias1,root.sg,INT32,TS_2DIFF,LZ4,{\"tag1\":\"v1\",\"tag2\":\"v2\"},null,null,null,BASE,", + "root.sg.d1.s2,null,root.sg,DOUBLE,GORILLA,LZ4,null,{\"attr3\":\"v3\"},null,null,BASE,", + "root.sg.d2.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d2.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d3.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,", + "root.sg.d3.s2,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,")); + checkResultSet(statement, "show timeseries root.sg.** where time!=1", expected); + checkResultSet( + statement, + "count timeseries root.sg.** where time!=1", + new HashSet<>(Collections.singletonList("6,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d0.s0,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d0.s1,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=2},BASE,", + "root.sg.d0.s2,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=0.01, compmintime=2, compmaxtime=15},BASE,")); + checkResultSet(statement, "show timeseries root.sg.** where time<2", expected); + checkResultSet( + statement, + "count timeseries root.sg.** where time<2", + new HashSet<>(Collections.singletonList("3,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d0.s0,null,root.sg,INT32,TS_2DIFF,LZ4,null,null,null,null,BASE,", + "root.sg.d0.s1,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=2},BASE,", + "root.sg.d0.s2,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=0.01, compmintime=2, compmaxtime=15},BASE,")); + checkResultSet(statement, "show timeseries root.sg.** where time=1", expected); + checkResultSet( + statement, + "count timeseries root.sg.** where time=1", + new HashSet<>(Collections.singletonList("3,"))); + // 3. Check non-root user + statement.execute("CREATE USER user1 'password'"); + statement.execute("GRANT READ_SCHEMA ON root.sg.d0.s1 TO USER user1"); + statement.execute("GRANT READ_SCHEMA ON root.sg.d3.s1 TO USER user1"); + try (Connection userCon = EnvFactory.getEnv().getConnection("user1", "password"); + Statement userStmt = userCon.createStatement()) { + // 3.1 Without read data permission, result set should be empty + try (ResultSet resultSet = + userStmt.executeQuery("show timeseries root.sg.** where time>0")) { + Assert.assertFalse(resultSet.next()); + } + checkResultSet( + userStmt, + "count timeseries root.sg.** where time>0", + new HashSet<>(Collections.singletonList("0,"))); + } + statement.execute("GRANT READ_DATA ON root.sg.d0.s1 TO USER user1"); + statement.execute("GRANT READ_DATA ON root.sg.d3.s1 TO USER user1"); + try (Connection userCon = EnvFactory.getEnv().getConnection("user1", "password"); + Statement userStmt = userCon.createStatement()) { + // 3.2 With read data permission, result set should not be empty + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d0.s1,null,root.sg,INT32,PLAIN,LZ4,null,null,SDT,{compdev=2},BASE,", + "root.sg.d3.s1,null,root.sg,INT64,RLE,SNAPPY,null,null,null,null,BASE,")); + checkResultSet(userStmt, "show timeseries root.sg.** where time>0", expected); + checkResultSet( + userStmt, + "count timeseries root.sg.** where time>0", + new HashSet<>(Collections.singletonList("2,"))); + } + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + @Test + public void testShowDevices() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + // 0. Check environment + Set expected = + new HashSet<>( + Arrays.asList( + "root.sg.d0,false,null,INF,", + "root.sg.d1,true,null,INF,", + "root.sg.d2,false,t1,INF,", + "root.sg.d3,true,t2,60000,")); + checkResultSet(statement, "show devices root.sg.*", expected); + // 1. Without any data, result set should be empty + try (ResultSet resultSet = statement.executeQuery("show devices root.sg.* where time>0")) { + Assert.assertFalse(resultSet.next()); + } + // 2. Insert data and check again. + // - root.sg.d0.* time=1 + // - root.sg.d1.* time=2 + // - root.sg.d2.* time=3 + // - root.sg.d3.* time=now() + statement.execute("insert into root.sg.d0(timestamp,s0, s1, s2) values(1,1,1,1)"); + statement.execute("insert into root.sg.d1(timestamp,s1,s2) values(2,2,2)"); + statement.execute("insert into root.sg.d2(timestamp,s1,s2) values(3,3,3)"); + statement.execute("insert into root.sg.d3(timestamp,s1,s2) values(now(),4,4)"); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d0,false,null,INF,", + "root.sg.d1,true,null,INF,", + "root.sg.d2,false,t1,INF,", + "root.sg.d3,true,t2,60000,")); + checkResultSet(statement, "show devices root.sg.* where time>0", expected); + checkResultSet( + statement, + "count devices root.sg.* where time>0", + new HashSet<>(Collections.singletonList("4,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1,true,null,INF,", + "root.sg.d2,false,t1,INF,", + "root.sg.d3,true,t2,60000,")); + checkResultSet(statement, "show devices root.sg.* where time>1", expected); + checkResultSet( + statement, + "count devices root.sg.* where time>1", + new HashSet<>(Collections.singletonList("3,"))); + expected = + new HashSet<>( + Arrays.asList( + "root.sg.d1,true,null,INF,", + "root.sg.d2,false,t1,INF,", + "root.sg.d3,true,t2,60000,")); + checkResultSet(statement, "show devices root.sg.* where time!=1", expected); + checkResultSet( + statement, + "count devices root.sg.* where time!=1", + new HashSet<>(Collections.singletonList("3,"))); + expected = new HashSet<>(Collections.singletonList("root.sg.d0,false,null,INF,")); + checkResultSet(statement, "show devices root.sg.* where time<2", expected); + checkResultSet( + statement, + "count devices root.sg.* where time<2", + new HashSet<>(Collections.singletonList("1,"))); + expected = new HashSet<>(Collections.singletonList("root.sg.d0,false,null,INF,")); + checkResultSet(statement, "show devices root.sg.* where time=1", expected); + checkResultSet( + statement, + "count devices root.sg.* where time=1", + new HashSet<>(Collections.singletonList("1,"))); + // 3. Check non-root user + statement.execute("CREATE USER user1 'password'"); + statement.execute("GRANT READ_SCHEMA ON root.sg.d0.s1 TO USER user1"); + statement.execute("GRANT READ_SCHEMA ON root.sg.d3.s1 TO USER user1"); + try (Connection userCon = EnvFactory.getEnv().getConnection("user1", "password"); + Statement userStmt = userCon.createStatement()) { + // 3.1 Without read data permission, result set should be empty + try (ResultSet resultSet = userStmt.executeQuery("show devices root.sg.* where time>0")) { + Assert.assertFalse(resultSet.next()); + } + checkResultSet( + userStmt, + "count devices root.sg.* where time>0", + new HashSet<>(Collections.singletonList("0,"))); + } + statement.execute("GRANT READ_DATA ON root.sg.d0.s1 TO USER user1"); + statement.execute("GRANT READ_DATA ON root.sg.d3.s1 TO USER user1"); + try (Connection userCon = EnvFactory.getEnv().getConnection("user1", "password"); + Statement userStmt = userCon.createStatement()) { + // 3.2 With read data permission, result set should not be empty + expected = + new HashSet<>(Arrays.asList("root.sg.d0,false,null,INF,", "root.sg.d3,true,t2,60000,")); + checkResultSet(userStmt, "show devices root.sg.* where time>0", expected); + checkResultSet( + userStmt, + "count devices root.sg.* where time>0", + new HashSet<>(Collections.singletonList("2,"))); + } + + } catch (Exception e) { + e.printStackTrace(); + Assert.fail(e.getMessage()); + } + } + + private void checkResultSet(Statement statement, String sql, Set expected) + throws Exception { + try (ResultSet resultSet = statement.executeQuery(sql)) { + ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); + while (resultSet.next()) { + StringBuilder builder = new StringBuilder(); + for (int i = 1; i <= resultSetMetaData.getColumnCount(); i++) { + builder.append(resultSet.getString(i)).append(","); + } + if (!expected.contains(builder.toString())) { + System.out.println(builder.toString()); + } + Assert.assertTrue(expected.contains(builder.toString())); + expected.remove(builder.toString()); + } + } + Assert.assertTrue(expected.isEmpty()); + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java b/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java index deefd0cccc90..1c3f3708ae37 100644 --- a/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/util/AbstractSchemaIT.java @@ -100,6 +100,14 @@ protected static void clearSchema() throws Exception { "DROP DEVICE TEMPLATE " + resultSet.getString(ColumnHeaderConstant.TEMPLATE_NAME)); } } + // drop all users + try (ResultSet resultSet = statement.executeQuery("LIST USER")) { + while (resultSet.next()) { + if (!resultSet.getString(ColumnHeaderConstant.USER).equals("root")) { + statement.execute("DROP USER " + resultSet.getString(ColumnHeaderConstant.USER)); + } + } + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/DeviceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/DeviceContext.java new file mode 100644 index 000000000000..3c6663c680b1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/DeviceContext.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.common; + +import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class DeviceContext { + private boolean isAligned; + private final int templateId; + + public DeviceContext(DeviceSchemaInfo deviceSchemaInfo) { + this.isAligned = deviceSchemaInfo.isAligned(); + this.templateId = deviceSchemaInfo.getTemplateId(); + } + + public DeviceContext(boolean isAligned, int templateId) { + this.isAligned = isAligned; + this.templateId = templateId; + } + + public boolean isAligned() { + return isAligned; + } + + public int getTemplateId() { + return templateId; + } + + public void serializeAttributes(ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(isAligned, byteBuffer); + ReadWriteIOUtils.write(templateId, byteBuffer); + } + + public void serializeAttributes(DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(isAligned, stream); + ReadWriteIOUtils.write(templateId, stream); + } + + public static DeviceContext deserialize(ByteBuffer buffer) { + boolean isAligned = ReadWriteIOUtils.readBool(buffer); + int templateId = ReadWriteIOUtils.readInt(buffer); + return new DeviceContext(isAligned, templateId); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + DeviceContext that = (DeviceContext) o; + return isAligned == that.isAligned && templateId == that.templateId; + } + + @Override + public int hashCode() { + return Objects.hash(isAligned, templateId); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java similarity index 76% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesSchemaInfo.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java index 946a3cfa9048..9225b97fc70a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/TimeseriesContext.java @@ -32,25 +32,24 @@ import static org.apache.iotdb.db.queryengine.execution.operator.schema.source.TimeSeriesSchemaSource.mapToString; -public class TimeseriesSchemaInfo { +public class TimeseriesContext { private final String dataType; private final String encoding; private final String compression; private final String tags; private final String alias; - - // TODO: Currently we can't get attributes from fetchSchema in query - // private final String attributes; + private final String attributes; private final String deadband; private final String deadbandParameters; - public TimeseriesSchemaInfo(IMeasurementSchemaInfo schemaInfo) { + public TimeseriesContext(IMeasurementSchemaInfo schemaInfo) { this.dataType = schemaInfo.getSchema().getType().toString(); this.encoding = schemaInfo.getSchema().getEncodingType().toString(); this.compression = schemaInfo.getSchema().getCompressor().toString(); this.alias = schemaInfo.getAlias(); this.tags = mapToString(schemaInfo.getTagMap()); + this.attributes = mapToString(schemaInfo.getAttributeMap()); Pair deadbandInfo = MetaUtils.parseDeadbandInfo(schemaInfo.getSchema().getProps()); this.deadband = deadbandInfo.left; @@ -77,6 +76,10 @@ public String getTags() { return tags; } + public String getAttributes() { + return attributes; + } + public String getDeadbandParameters() { return deadbandParameters; } @@ -85,12 +88,13 @@ public String getDeadband() { return deadband; } - public TimeseriesSchemaInfo( + public TimeseriesContext( String dataType, String alias, String encoding, String compression, String tags, + String attributes, String deadband, String deadbandParameters) { this.dataType = dataType; @@ -98,6 +102,7 @@ public TimeseriesSchemaInfo( this.encoding = encoding; this.compression = compression; this.tags = tags; + this.attributes = attributes; this.deadband = deadband; this.deadbandParameters = deadbandParameters; } @@ -108,6 +113,7 @@ public void serializeAttributes(ByteBuffer byteBuffer) { ReadWriteIOUtils.write(encoding, byteBuffer); ReadWriteIOUtils.write(compression, byteBuffer); ReadWriteIOUtils.write(tags, byteBuffer); + ReadWriteIOUtils.write(attributes, byteBuffer); ReadWriteIOUtils.write(deadband, byteBuffer); ReadWriteIOUtils.write(deadbandParameters, byteBuffer); } @@ -118,20 +124,22 @@ public void serializeAttributes(DataOutputStream stream) throws IOException { ReadWriteIOUtils.write(encoding, stream); ReadWriteIOUtils.write(compression, stream); ReadWriteIOUtils.write(tags, stream); + ReadWriteIOUtils.write(attributes, stream); ReadWriteIOUtils.write(deadband, stream); ReadWriteIOUtils.write(deadbandParameters, stream); } - public static TimeseriesSchemaInfo deserialize(ByteBuffer buffer) { + public static TimeseriesContext deserialize(ByteBuffer buffer) { String dataType = ReadWriteIOUtils.readString(buffer); String alias = ReadWriteIOUtils.readString(buffer); String encoding = ReadWriteIOUtils.readString(buffer); String compression = ReadWriteIOUtils.readString(buffer); String tags = ReadWriteIOUtils.readString(buffer); + String attributes = ReadWriteIOUtils.readString(buffer); String deadband = ReadWriteIOUtils.readString(buffer); String deadbandParameters = ReadWriteIOUtils.readString(buffer); - return new TimeseriesSchemaInfo( - dataType, alias, encoding, compression, tags, deadband, deadbandParameters); + return new TimeseriesContext( + dataType, alias, encoding, compression, tags, attributes, deadband, deadbandParameters); } @Override @@ -142,18 +150,22 @@ public boolean equals(Object obj) { if (obj == null || getClass() != obj.getClass()) { return false; } - TimeseriesSchemaInfo that = (TimeseriesSchemaInfo) obj; - return Objects.equals(dataType, that.dataType) - && Objects.equals(alias, that.alias) - && encoding.equals(that.encoding) - && Objects.equals(compression, that.compression) - && Objects.equals(tags, that.tags) - && Objects.equals(deadband, that.deadband) - && Objects.equals(deadbandParameters, that.deadbandParameters); + TimeseriesContext that = (TimeseriesContext) obj; + boolean res = + Objects.equals(dataType, that.dataType) + && Objects.equals(alias, that.alias) + && encoding.equals(that.encoding) + && Objects.equals(compression, that.compression) + && Objects.equals(tags, that.tags) + && Objects.equals(attributes, that.attributes) + && Objects.equals(deadband, that.deadband) + && Objects.equals(deadbandParameters, that.deadbandParameters); + return res; } @Override public int hashCode() { - return Objects.hash(dataType, alias, encoding, compression, tags, deadband, deadbandParameters); + return Objects.hash( + dataType, alias, encoding, compression, tags, attributes, deadband, deadbandParameters); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java index c461d0da83ff..dc9776ab54ab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTree.java @@ -147,7 +147,8 @@ private IMeasurementSchemaInfo getMeasurementSchemaInfo( } else { Template template = templateMap.get(entityNode.getTemplateId()); if (template != null && template.getSchemaMap().containsKey(measurement)) { - return new MeasurementSchemaInfo(measurement, template.getSchema(measurement), null, null); + return new MeasurementSchemaInfo( + measurement, template.getSchema(measurement), null, null, null); } } return null; @@ -259,7 +260,8 @@ public void computeSourceOfLogicalView( measurementPath.getMeasurement(), measurementPath.getMeasurementSchema(), null, - measurementPath.getTagMap()), + measurementPath.getTagMap(), + null), measurementPath.isUnderAlignedEntity()); } } @@ -303,7 +305,9 @@ public void appendTemplateDevice( entityNode.setTemplateId(templateId); cur.replaceChild(deviceName, entityNode); } - templateMap.putIfAbsent(templateId, template); + if (template != null) { + templateMap.putIfAbsent(templateId, template); + } } public void appendMeasurementPaths(List measurementPathList) { @@ -317,6 +321,7 @@ public void appendSingleMeasurementPath(MeasurementPath measurementPath) { measurementPath, measurementPath.getMeasurementSchema(), measurementPath.getTagMap(), + null, measurementPath.isMeasurementAliasExists() ? measurementPath.getMeasurementAlias() : null, measurementPath.isUnderAlignedEntity()); } @@ -325,6 +330,7 @@ public void appendSingleMeasurement( PartialPath path, IMeasurementSchema schema, Map tagMap, + Map attributeMap, String alias, boolean isAligned) { String[] nodes = path.getNodes(); @@ -340,6 +346,7 @@ public void appendSingleMeasurement( cur.getAsEntityNode().addAliasChild(alias, measurementNode); } measurementNode.setTagMap(tagMap); + measurementNode.setAttributeMap(attributeMap); child = measurementNode; if (schema.isLogicalView()) { this.hasLogicalMeasurementPath = true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/IMeasurementSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/IMeasurementSchemaInfo.java index 7200c9a5e5f2..5636363f800f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/IMeasurementSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/IMeasurementSchemaInfo.java @@ -43,6 +43,8 @@ public interface IMeasurementSchemaInfo { Map getTagMap(); + Map getAttributeMap(); + String getAlias(); boolean isLogicalView(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/MeasurementSchemaInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/MeasurementSchemaInfo.java index a8225dbcb0ad..9db3ebadbfde 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/MeasurementSchemaInfo.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/MeasurementSchemaInfo.java @@ -37,14 +37,20 @@ public class MeasurementSchemaInfo implements IMeasurementSchemaInfo { private final String name; private final String alias; private Map tagMap; + private Map attributeMap; private final IMeasurementSchema schema; public MeasurementSchemaInfo( - String name, IMeasurementSchema schema, String alias, Map tagMap) { + String name, + IMeasurementSchema schema, + String alias, + Map tagMap, + Map attributeMap) { this.name = name; this.schema = schema; this.alias = alias; this.tagMap = tagMap; + this.attributeMap = attributeMap; } public String getName() { @@ -68,6 +74,11 @@ public Map getTagMap() { return tagMap; } + @Override + public Map getAttributeMap() { + return attributeMap; + } + @Override public LogicalViewSchema getSchemaAsLogicalViewSchema() { if (this.isLogicalView()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/node/SchemaMeasurementNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/node/SchemaMeasurementNode.java index 55657a59700b..b1eaa7ff5c10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/node/SchemaMeasurementNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/node/SchemaMeasurementNode.java @@ -36,8 +36,8 @@ public class SchemaMeasurementNode extends SchemaNode implements IMeasurementSch private String alias; private IMeasurementSchema schema; - private Map tagMap; + private Map attributeMap; public SchemaMeasurementNode(String name, IMeasurementSchema schema) { super(name); @@ -82,6 +82,15 @@ public Map getTagMap() { return tagMap; } + @Override + public Map getAttributeMap() { + return attributeMap; + } + + public void setAttributeMap(Map attributeMap) { + this.attributeMap = attributeMap; + } + @Override public SchemaNode getChild(String name) { return null; @@ -144,6 +153,7 @@ public void serialize(OutputStream outputStream) throws IOException { schema.serializeTo(outputStream); ReadWriteIOUtils.write(tagMap, outputStream); + ReadWriteIOUtils.write(attributeMap, outputStream); } public static SchemaMeasurementNode deserialize(InputStream inputStream) throws IOException { @@ -161,10 +171,12 @@ public static SchemaMeasurementNode deserialize(InputStream inputStream) throws } Map tagMap = ReadWriteIOUtils.readMap(inputStream); + Map attributeMap = ReadWriteIOUtils.readMap(inputStream); SchemaMeasurementNode measurementNode = new SchemaMeasurementNode(name, schema); measurementNode.setAlias(alias); measurementNode.setTagMap(tagMap); + measurementNode.setAttributeMap(attributeMap); return measurementNode; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeDeviceVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeDeviceVisitor.java index 42fe1b15db19..684035cce16f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeDeviceVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/schematree/visitor/SchemaTreeDeviceVisitor.java @@ -67,7 +67,8 @@ protected DeviceSchemaInfo generateResult(SchemaNode nextMatchedNode) { measurementNode.getName(), measurementNode.getSchema(), measurementNode.getAlias(), - measurementNode.getTagMap())); + measurementNode.getTagMap(), + measurementNode.getAttributeMap())); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java index c30c11549102..3fc9598ef8af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/driver/DataDriverContext.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.operator.source.DataSourceOperator; import org.apache.iotdb.db.storageengine.dataregion.IDataRegionForQuery; @@ -39,7 +40,7 @@ public class DataDriverContext extends DriverContext { // it will be set to null, after being merged into Parent FIContext private List paths; private QueryDataSourceType queryDataSourceType = null; - private Map deviceIDToAligned; + private Map deviceIDToContext; // it will be set to null, after QueryDataSource being inited private List sourceOperators; @@ -47,27 +48,27 @@ public DataDriverContext(FragmentInstanceContext fragmentInstanceContext, int pi super(fragmentInstanceContext, pipelineId); this.paths = new ArrayList<>(); this.sourceOperators = new ArrayList<>(); - this.deviceIDToAligned = null; + this.deviceIDToContext = null; } public DataDriverContext(DataDriverContext parentContext, int pipelineId) { super(parentContext.getFragmentInstanceContext(), pipelineId); this.paths = new ArrayList<>(); this.sourceOperators = new ArrayList<>(); - this.deviceIDToAligned = null; + this.deviceIDToContext = null; } public void setQueryDataSourceType(QueryDataSourceType queryDataSourceType) { this.queryDataSourceType = queryDataSourceType; } - public void setDeviceIDToAligned(Map deviceIDToAligned) { - this.deviceIDToAligned = deviceIDToAligned; + public void setDeviceIDToContext(Map deviceIDToContext) { + this.deviceIDToContext = deviceIDToContext; } - public void clearDeviceIDToAligned() { + public void clearDeviceIDToContext() { // friendly for gc - deviceIDToAligned = null; + deviceIDToContext = null; } public void addPath(PartialPath path) { @@ -82,8 +83,8 @@ public List getPaths() { return paths; } - public Map getDeviceIDToAligned() { - return deviceIDToAligned; + public Map getDeviceIDToContext() { + return deviceIDToContext; } public Optional getQueryDataSourceType() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 72425e4df363..3748372c4f61 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.common.QueryId; import org.apache.iotdb.db.queryengine.common.SessionInfo; @@ -68,7 +69,7 @@ public class FragmentInstanceContext extends QueryContext { // it will only be used once, after sharedQueryDataSource being inited, it will be set to null private List sourcePaths; // Used for region scan. - private Map devicePathsToAligned; + private Map devicePathsToContext; // Shared by all scan operators in this fragment instance to avoid memory problem private IQueryDataSource sharedQueryDataSource; @@ -356,8 +357,8 @@ public void setSourcePaths(List sourcePaths) { this.sourcePaths = sourcePaths; } - public void setDevicePathsToAligned(Map devicePathsToAligned) { - this.devicePathsToAligned = devicePathsToAligned; + public void setDevicePathsToContext(Map devicePathsToContext) { + this.devicePathsToContext = devicePathsToContext; } public void initQueryDataSource(List sourcePaths) throws QueryProcessException { @@ -399,17 +400,17 @@ public void initQueryDataSource(List sourcePaths) throws QueryProce } } - public void initRegionScanQueryDataSource(Map devicePathToAligned) + public void initRegionScanQueryDataSource(Map devicePathsToContext) throws QueryProcessException { long startTime = System.nanoTime(); - if (devicePathsToAligned == null) { + if (devicePathsToContext == null) { return; } dataRegion.readLock(); try { this.sharedQueryDataSource = dataRegion.queryForDeviceRegionScan( - devicePathToAligned, + devicePathsToContext, this, globalTimeFilter != null ? globalTimeFilter.copy() : null, timePartitions); @@ -460,8 +461,8 @@ public synchronized IQueryDataSource getSharedQueryDataSource() throws QueryProc sourcePaths = null; break; case DEVICE_REGION_SCAN: - initRegionScanQueryDataSource(devicePathsToAligned); - devicePathsToAligned = null; + initRegionScanQueryDataSource(devicePathsToContext); + devicePathsToContext = null; break; case TIME_SERIES_REGION_SCAN: initRegionScanQueryDataSource(sourcePaths); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java index c6236d018a98..f4b40c69b79e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperator.java @@ -22,6 +22,7 @@ import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.exception.runtime.SchemaExecutionException; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; @@ -40,6 +41,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.util.Collections; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; @@ -52,8 +54,13 @@ public class SchemaFetchScanOperator implements SourceOperator { private final ISchemaRegion schemaRegion; private final boolean withTags; + private final boolean withAttributes; private final boolean withTemplate; + private final boolean withAliasForce; + + private final boolean fetchDevice; private boolean isFinished = false; + private final PathPatternTree authorityScope; private static final int DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES = TSFileDescriptor.getInstance().getConfig().getMaxTsBlockSizeInBytes(); @@ -61,21 +68,78 @@ public class SchemaFetchScanOperator implements SourceOperator { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(SchemaFetchScanOperator.class); - public SchemaFetchScanOperator( + public static SchemaFetchScanOperator ofSeries( PlanNodeId planNodeId, OperatorContext context, PathPatternTree patternTree, Map templateMap, ISchemaRegion schemaRegion, boolean withTags, - boolean withTemplate) { + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) { + return new SchemaFetchScanOperator( + planNodeId, + context, + patternTree, + templateMap, + schemaRegion, + withTags, + withAttributes, + withTemplate, + withAliasForce); + } + + public static SchemaFetchScanOperator ofDevice( + PlanNodeId planNodeId, + OperatorContext context, + PathPatternTree patternTree, + PathPatternTree authorityScope, + ISchemaRegion schemaRegion) { + return new SchemaFetchScanOperator( + planNodeId, context, patternTree, authorityScope, schemaRegion); + } + + private SchemaFetchScanOperator( + PlanNodeId planNodeId, + OperatorContext context, + PathPatternTree patternTree, + Map templateMap, + ISchemaRegion schemaRegion, + boolean withTags, + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) { this.sourceId = planNodeId; this.operatorContext = context; this.patternTree = patternTree; this.schemaRegion = schemaRegion; this.templateMap = templateMap; this.withTags = withTags; + this.withAttributes = withAttributes; this.withTemplate = withTemplate; + this.withAliasForce = withAliasForce; + this.fetchDevice = false; + this.authorityScope = SchemaConstant.ALL_MATCH_SCOPE; + } + + private SchemaFetchScanOperator( + PlanNodeId planNodeId, + OperatorContext context, + PathPatternTree patternTree, + PathPatternTree authorityScope, + ISchemaRegion schemaRegion) { + this.sourceId = planNodeId; + this.operatorContext = context; + this.patternTree = patternTree; + this.schemaRegion = schemaRegion; + this.templateMap = Collections.emptyMap(); + this.withTags = false; + this.withAttributes = false; + this.withTemplate = false; + this.withAliasForce = false; + this.fetchDevice = true; + this.authorityScope = authorityScope; } @Override @@ -118,7 +182,10 @@ public PlanNodeId getSourceId() { private TsBlock fetchSchema() throws MetadataException { ClusterSchemaTree schemaTree = - schemaRegion.fetchSchema(patternTree, templateMap, withTags, withTemplate); + fetchDevice + ? schemaRegion.fetchDeviceSchema(patternTree, authorityScope) + : schemaRegion.fetchSeriesSchema( + patternTree, templateMap, withTags, withAttributes, withTemplate, withAliasForce); ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceSchemaSource.java index 37a9d63865c7..d1f6029f04c5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceSchemaSource.java @@ -109,7 +109,7 @@ public void transformToTsBlockColumns( builder .getColumnBuilder(2) .writeBinary(new Binary(String.valueOf(device.isAligned()), TSFileConfig.STRING_CHARSET)); - if (templateId != -1) { + if (templateId != SchemaConstant.NON_TEMPLATE) { builder .getColumnBuilder(3) .writeBinary( @@ -125,7 +125,7 @@ public void transformToTsBlockColumns( builder .getColumnBuilder(1) .writeBinary(new Binary(String.valueOf(device.isAligned()), TSFileConfig.STRING_CHARSET)); - if (templateId != -1) { + if (templateId != SchemaConstant.NON_TEMPLATE) { builder .getColumnBuilder(2) .writeBinary( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java index 873352b8b741..6437fe673faf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java @@ -19,11 +19,16 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; +import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; +import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.schemaengine.template.ClusterTemplateManager; import org.apache.tsfile.block.column.ColumnBuilder; import org.apache.tsfile.common.conf.TSFileConfig; @@ -41,7 +46,7 @@ public class ActiveDeviceRegionScanOperator extends AbstractRegionScanDataSourceOperator { // The devices which need to be checked. - private final Map deviceToAlignedMap; + private final Map deviceContextMap; private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ActiveDeviceRegionScanOperator.class) @@ -50,24 +55,24 @@ public class ActiveDeviceRegionScanOperator extends AbstractRegionScanDataSource public ActiveDeviceRegionScanOperator( OperatorContext operatorContext, PlanNodeId sourceId, - Map deviceToAlignedMap, + Map deviceContextMap, Filter timeFilter, boolean outputCount) { this.outputCount = outputCount; this.sourceId = sourceId; this.operatorContext = operatorContext; - this.deviceToAlignedMap = deviceToAlignedMap; + this.deviceContextMap = deviceContextMap; this.regionScanUtil = new RegionScanForActiveDeviceUtil(timeFilter); } @Override protected boolean getNextTsFileHandle() throws IOException, IllegalPathException { - return ((RegionScanForActiveDeviceUtil) regionScanUtil).nextTsFileHandle(deviceToAlignedMap); + return ((RegionScanForActiveDeviceUtil) regionScanUtil).nextTsFileHandle(deviceContextMap); } @Override protected boolean isAllDataChecked() { - return deviceToAlignedMap.isEmpty(); + return deviceContextMap.isEmpty(); } @Override @@ -77,20 +82,38 @@ protected void updateActiveData() { if (this.outputCount) { count += activeDevices.size(); - activeDevices.forEach(deviceToAlignedMap.keySet()::remove); + activeDevices.forEach(deviceContextMap.keySet()::remove); } else { TimeColumnBuilder timeColumnBuilder = resultTsBlockBuilder.getTimeColumnBuilder(); ColumnBuilder[] columnBuilders = resultTsBlockBuilder.getValueColumnBuilders(); for (IDeviceID deviceID : activeDevices) { + DeviceContext deviceContext = deviceContextMap.get(deviceID); + int templateId = deviceContext.getTemplateId(); + // TODO: use IDeviceID interface to get ttl + long ttl = DataNodeTTLCache.getInstance().getTTL(deviceID); + + // TODO: make it more readable, like "30 days" or "10 hours" + String ttlStr = ttl == Long.MAX_VALUE ? IoTDBConstant.TTL_INFINITE : String.valueOf(ttl); + timeColumnBuilder.writeLong(-1); columnBuilders[0].writeBinary(new Binary(deviceID.getBytes())); columnBuilders[1].writeBinary( new Binary( - String.valueOf(deviceToAlignedMap.get(deviceID)), TSFileConfig.STRING_CHARSET)); - columnBuilders[2].appendNull(); - columnBuilders[3].appendNull(); + String.valueOf(deviceContextMap.get(deviceID).isAligned()), + TSFileConfig.STRING_CHARSET)); + + if (templateId != SchemaConstant.NON_TEMPLATE) { + columnBuilders[2].writeBinary( + new Binary( + String.valueOf( + ClusterTemplateManager.getInstance().getTemplate(templateId).getName()), + TSFileConfig.STRING_CHARSET)); + } else { + columnBuilders[2].appendNull(); + } + columnBuilders[3].writeBinary(new Binary(ttlStr, TSFileConfig.STRING_CHARSET)); resultTsBlockBuilder.declarePosition(); - deviceToAlignedMap.remove(deviceID); + deviceContextMap.remove(deviceID); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java index e3385f92ecd0..b947bfcee89e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext; @@ -42,7 +42,7 @@ public class ActiveTimeSeriesRegionScanOperator extends AbstractRegionScanDataSourceOperator { // Timeseries which need to be checked. - private final Map> timeSeriesToSchemasInfo; + private final Map> timeSeriesToSchemasInfo; private static final Binary VIEW_TYPE = new Binary("BASE".getBytes()); private final Binary dataBaseName; private static final long INSTANCE_SIZE = @@ -53,7 +53,7 @@ public class ActiveTimeSeriesRegionScanOperator extends AbstractRegionScanDataSo public ActiveTimeSeriesRegionScanOperator( OperatorContext operatorContext, PlanNodeId sourceId, - Map> timeSeriesToSchemasInfo, + Map> timeSeriesToSchemasInfo, Filter timeFilter, boolean isOutputCount) { this.outputCount = isOutputCount; @@ -111,9 +111,9 @@ protected void updateActiveData() { IDeviceID deviceID = entry.getKey(); String deviceStr = ((PlainDeviceID) deviceID).toStringID(); List timeSeriesList = entry.getValue(); - Map timeSeriesInfo = timeSeriesToSchemasInfo.get(deviceID); + Map timeSeriesInfo = timeSeriesToSchemasInfo.get(deviceID); for (String timeSeries : timeSeriesList) { - TimeseriesSchemaInfo schemaInfo = timeSeriesInfo.get(timeSeries); + TimeseriesContext schemaInfo = timeSeriesInfo.get(timeSeries); timeColumnBuilder.writeLong(-1); columnBuilders[0].writeBinary( new Binary(contactDeviceAndMeasurement(deviceStr, timeSeries))); @@ -124,7 +124,7 @@ protected void updateActiveData() { checkAndAppend(schemaInfo.getEncoding(), columnBuilders[4]); // Encoding checkAndAppend(schemaInfo.getCompression(), columnBuilders[5]); // Compression checkAndAppend(schemaInfo.getTags(), columnBuilders[6]); // Tags - columnBuilders[7].appendNull(); // Attributes + checkAndAppend(schemaInfo.getAttributes(), columnBuilders[7]); // Attributes checkAndAppend(schemaInfo.getDeadband(), columnBuilders[8]); // Description checkAndAppend(schemaInfo.getDeadbandParameters(), columnBuilders[9]); // DeadbandParameters columnBuilders[10].writeBinary(VIEW_TYPE); // ViewType @@ -135,7 +135,7 @@ protected void updateActiveData() { } private void removeTimeseriesListFromDevice(IDeviceID deviceID, List timeSeriesList) { - Map timeSeriesInfo = timeSeriesToSchemasInfo.get(deviceID); + Map timeSeriesInfo = timeSeriesToSchemasInfo.get(deviceID); for (String timeSeries : timeSeriesList) { timeSeriesInfo.remove(timeSeries); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java index 8ca5e165363f..11faa55f3941 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractChunkOffset; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractDeviceChunkMetaData; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.DeviceStartEndTime; @@ -60,7 +61,7 @@ public boolean isCurrentTsFileFinished() { return deviceSetForCurrentTsFile.isEmpty(); } - public boolean nextTsFileHandle(Map targetDevices) + public boolean nextTsFileHandle(Map targetDevices) throws IOException, IllegalPathException { if (!queryDataSource.hasNext()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java index ff6711232bba..61c7028ab211 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveTimeSeriesUtil.java @@ -20,7 +20,7 @@ package org.apache.iotdb.db.queryengine.execution.operator.source; import org.apache.iotdb.commons.exception.IllegalPathException; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractChunkOffset; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractDeviceChunkMetaData; import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.DeviceStartEndTime; @@ -55,8 +55,8 @@ public RegionScanForActiveTimeSeriesUtil(Filter timeFilter) { this.activeTimeSeries = new HashMap<>(); } - public boolean nextTsFileHandle( - Map> targetTimeseries) throws IOException { + public boolean nextTsFileHandle(Map> targetTimeseries) + throws IOException { if (!queryDataSource.hasNext()) { // There is no more TsFileHandles to be scanned. return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java index 2aeec556fac3..94a1e518b74b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java @@ -28,9 +28,10 @@ import org.apache.iotdb.commons.partition.DataPartition; import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.NodeRef; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; import org.apache.iotdb.db.queryengine.plan.execution.memory.StatementMemorySource; @@ -303,23 +304,23 @@ aggregation results last_value(temperature) and last_value(status), whereas buck private List measurementSchemaList; // Used for regionScan - private Map devicePathToAlignedStatus; - private Map>> deviceToTimeseriesSchemas; + private Map devicePathToContextMap; + private Map>> deviceToTimeseriesSchemas; - public void setDevicePathToAlignedStatus(Map devicePathToAlignedStatus) { - this.devicePathToAlignedStatus = devicePathToAlignedStatus; + public void setDevicePathToContextMap(Map devicePathToContextMap) { + this.devicePathToContextMap = devicePathToContextMap; } - public Map getDevicePathToAlignedStatus() { - return devicePathToAlignedStatus; + public Map getDevicePathToContextMap() { + return devicePathToContextMap; } public void setDeviceToTimeseriesSchemas( - Map>> deviceToTimeseriesSchemas) { + Map>> deviceToTimeseriesSchemas) { this.deviceToTimeseriesSchemas = deviceToTimeseriesSchemas; } - public Map>> + public Map>> getDeviceToTimeseriesSchemas() { return deviceToTimeseriesSchemas; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java index c6f7b240fd8f..6d5058e5417a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/AnalyzeVisitor.java @@ -48,8 +48,9 @@ import org.apache.iotdb.db.protocol.client.ConfigNodeClient; import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; @@ -106,10 +107,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.DeviceSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; -import org.apache.iotdb.db.queryengine.plan.statement.internal.SchemaFetchStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.SeriesSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDevicesStatement; @@ -2883,20 +2885,27 @@ private boolean analyzeTimeseriesRegionScan( WhereCondition timeCondition, PathPatternTree patternTree, Analysis analysis, - MPPQueryContext context) + MPPQueryContext context, + PathPatternTree authorityScope) throws IllegalPathException { analyzeGlobalTimeConditionInShowMetaData(timeCondition, analysis); context.generateGlobalTimeFilter(analysis); + patternTree.constructTree(); + ISchemaTree schemaTree = + schemaFetcher.fetchRawSchemaInMeasurementLevel(patternTree, authorityScope, context); - ISchemaTree schemaTree = schemaFetcher.fetchSchemaWithTags(patternTree, false, context); if (schemaTree.isEmpty()) { analysis.setFinishQueryAfterAnalyze(true); return false; } removeLogicViewMeasurement(schemaTree); - - Map>> deviceToTimeseriesSchemaInfo = + Map>> deviceToTimeseriesContext = new HashMap<>(); + /** + * Since we fetch raw time series schema without template(The template sequence will be treated + * as a normal node, not a device+templateId. This means that all nodes are what we need.). We + * can use ALL_MATCH_PATTERN to get result. + */ List deviceSchemaInfoList = schemaTree.getMatchedDevices(ALL_MATCH_PATTERN); Set deviceSet = new HashSet<>(); for (DeviceSchemaInfo deviceSchemaInfo : deviceSchemaInfoList) { @@ -2906,33 +2915,34 @@ private boolean analyzeTimeseriesRegionScan( if (isAligned) { List measurementList = new ArrayList<>(); List schemaList = new ArrayList<>(); - List timeseriesSchemaInfoList = new ArrayList<>(); + List timeseriesContextList = new ArrayList<>(); for (IMeasurementSchemaInfo measurementSchemaInfo : deviceSchemaInfo.getMeasurementSchemaInfoList()) { schemaList.add(measurementSchemaInfo.getSchema()); measurementList.add(measurementSchemaInfo.getName()); - timeseriesSchemaInfoList.add(new TimeseriesSchemaInfo(measurementSchemaInfo)); + timeseriesContextList.add(new TimeseriesContext(measurementSchemaInfo)); } AlignedPath alignedPath = new AlignedPath(devicePath.getNodes(), measurementList, schemaList); - deviceToTimeseriesSchemaInfo + deviceToTimeseriesContext .computeIfAbsent(devicePath, k -> new HashMap<>()) - .put(alignedPath, timeseriesSchemaInfoList); + .put(alignedPath, timeseriesContextList); } else { for (IMeasurementSchemaInfo measurementSchemaInfo : deviceSchemaInfo.getMeasurementSchemaInfoList()) { MeasurementPath measurementPath = new MeasurementPath( devicePath.concatNode(measurementSchemaInfo.getName()).getNodes()); - deviceToTimeseriesSchemaInfo + deviceToTimeseriesContext .computeIfAbsent(devicePath, k -> new HashMap<>()) .put( measurementPath, - Collections.singletonList(new TimeseriesSchemaInfo(measurementSchemaInfo))); + Collections.singletonList(new TimeseriesContext(measurementSchemaInfo))); } } } - analysis.setDeviceToTimeseriesSchemas(deviceToTimeseriesSchemaInfo); + + analysis.setDeviceToTimeseriesSchemas(deviceToTimeseriesContext); // fetch Data partition DataPartition dataPartition = fetchDataPartitionByDevices(deviceSet, schemaTree, context); analysis.setDataPartitionInfo(dataPartition); @@ -2953,7 +2963,11 @@ public Analysis visitShowTimeSeries( // If there is time condition in SHOW TIMESERIES, we need to scan the raw data boolean hasSchema = analyzeTimeseriesRegionScan( - showTimeSeriesStatement.getTimeCondition(), patternTree, analysis, context); + showTimeSeriesStatement.getTimeCondition(), + patternTree, + analysis, + context, + showTimeSeriesStatement.getAuthorityScope()); if (!hasSchema) { analysis.setRespDatasetHeader(DatasetHeaderFactory.getShowTimeSeriesHeader()); return analysis; @@ -3037,29 +3051,31 @@ private void removeLogicViewMeasurement(ISchemaTree schemaTree) { private void analyzeDeviceRegionScan( WhereCondition timeCondition, - PathPatternTree patternTree, + PartialPath pattern, + PathPatternTree authorityScope, Analysis analysis, MPPQueryContext context) { // If there is time condition in SHOW DEVICES, we need to scan the raw data analyzeGlobalTimeConditionInShowMetaData(timeCondition, analysis); context.generateGlobalTimeFilter(analysis); - - ISchemaTree schemaTree = schemaFetcher.fetchSchemaInDeviceLevel(patternTree, context); + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern(pattern); + ISchemaTree schemaTree = + schemaFetcher.fetchRawSchemaInDeviceLevel(patternTree, authorityScope, context); if (schemaTree.isEmpty()) { analysis.setFinishQueryAfterAnalyze(true); return; } // fetch Data partition - List deviceSchemaInfoList = schemaTree.getMatchedDevices(ALL_MATCH_PATTERN); - Map devicePathsToAlignedStatus = new HashMap<>(); - for (DeviceSchemaInfo deviceSchema : deviceSchemaInfoList) { - devicePathsToAlignedStatus.put(deviceSchema.getDevicePath(), deviceSchema.isAligned()); - } - analysis.setDevicePathToAlignedStatus(devicePathsToAlignedStatus); + + Map devicePathsToInfoMap = + schemaTree.getMatchedDevices(pattern).stream() + .collect(Collectors.toMap(DeviceSchemaInfo::getDevicePath, DeviceContext::new)); + analysis.setDevicePathToContextMap(devicePathsToInfoMap); DataPartition dataPartition = fetchDataPartitionByDevices( - devicePathsToAlignedStatus.keySet().stream() + devicePathsToInfoMap.keySet().stream() .map(PartialPath::getFullPath) .collect(Collectors.toSet()), schemaTree, @@ -3073,14 +3089,17 @@ public Analysis visitShowDevices( Analysis analysis = new Analysis(); analysis.setStatement(showDevicesStatement); - PathPatternTree patternTree = new PathPatternTree(); - patternTree.appendPathPattern( - showDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); - if (showDevicesStatement.hasTimeCondition()) { analyzeDeviceRegionScan( - showDevicesStatement.getTimeCondition(), patternTree, analysis, context); + showDevicesStatement.getTimeCondition(), + showDevicesStatement.getPathPattern(), + showDevicesStatement.getAuthorityScope(), + analysis, + context); } else { + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern( + showDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); SchemaPartition schemaPartitionInfo = partitionFetcher.getSchemaPartition(patternTree); analysis.setSchemaPartitionInfo(schemaPartitionInfo); } @@ -3114,13 +3133,34 @@ public Analysis visitCountStorageGroup( } @Override - public Analysis visitSchemaFetch( - SchemaFetchStatement schemaFetchStatement, MPPQueryContext context) { + public Analysis visitSeriesSchemaFetch( + SeriesSchemaFetchStatement seriesSchemaFetchStatement, MPPQueryContext context) { Analysis analysis = new Analysis(); - analysis.setStatement(schemaFetchStatement); + analysis.setStatement(seriesSchemaFetchStatement); SchemaPartition schemaPartition = - partitionFetcher.getSchemaPartition(schemaFetchStatement.getPatternTree()); + partitionFetcher.getSchemaPartition(seriesSchemaFetchStatement.getPatternTree()); + analysis.setSchemaPartitionInfo(schemaPartition); + + if (schemaPartition.isEmpty()) { + analysis.setFinishQueryAfterAnalyze(true); + } + + return analysis; + } + + @Override + public Analysis visitDeviceSchemaFetch( + DeviceSchemaFetchStatement deviceSchemaFetchStatement, MPPQueryContext context) { + Analysis analysis = new Analysis(); + analysis.setStatement(deviceSchemaFetchStatement); + + PathPatternTree patternTree = new PathPatternTree(); + for (PartialPath path : deviceSchemaFetchStatement.getPaths()) { + patternTree.appendPathPattern(path); + } + patternTree.constructTree(); + SchemaPartition schemaPartition = partitionFetcher.getSchemaPartition(patternTree); analysis.setSchemaPartitionInfo(schemaPartition); if (schemaPartition.isEmpty()) { @@ -3136,13 +3176,17 @@ public Analysis visitCountDevices( Analysis analysis = new Analysis(); analysis.setStatement(countDevicesStatement); - PathPatternTree patternTree = new PathPatternTree(); - patternTree.appendPathPattern( - countDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); if (countDevicesStatement.hasTimeCondition()) { analyzeDeviceRegionScan( - countDevicesStatement.getTimeCondition(), patternTree, analysis, context); + countDevicesStatement.getTimeCondition(), + countDevicesStatement.getPathPattern(), + countDevicesStatement.getAuthorityScope(), + analysis, + context); } else { + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern( + countDevicesStatement.getPathPattern().concatNode(IoTDBConstant.ONE_LEVEL_PATH_WILDCARD)); SchemaPartition schemaPartitionInfo = partitionFetcher.getSchemaPartition(patternTree); analysis.setSchemaPartitionInfo(schemaPartitionInfo); } @@ -3164,7 +3208,11 @@ public Analysis visitCountTimeSeries( try { boolean hasSchema = analyzeTimeseriesRegionScan( - countTimeSeriesStatement.getTimeCondition(), patternTree, analysis, context); + countTimeSeriesStatement.getTimeCondition(), + patternTree, + analysis, + context, + countTimeSeriesStatement.getAuthorityScope()); if (!hasSchema) { analysis.setRespDatasetHeader(DatasetHeaderFactory.getCountTimeSeriesHeader()); return analysis; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java index bc51cae2c864..10395b258f64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java @@ -23,6 +23,9 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.PlainDeviceID; + import java.util.concurrent.locks.ReentrantReadWriteLock; public class DataNodeTTLCache { @@ -70,6 +73,15 @@ public void unsetTTL(String[] path) { } } + // TODO: support real implementation + public long getTTL(IDeviceID deviceID) { + try { + return getTTL(((PlainDeviceID) deviceID).toStringID()); + } catch (IllegalPathException e) { + return Long.MAX_VALUE; + } + } + public long getTTL(String path) throws IllegalPathException { lock.readLock().lock(); try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java index 1ea59da13554..167242ff20cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DeviceUsingTemplateSchemaCache.java @@ -168,6 +168,11 @@ public Map getTagMap() { return null; } + @Override + public Map getAttributeMap() { + return null; + } + @Override public String getAlias() { return null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java index d3d99c2548be..bf6d5c31a86c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/SchemaCacheEntry.java @@ -70,6 +70,11 @@ public Map getTagMap() { return tagMap; } + @Override + public Map getAttributeMap() { + return null; + } + public TSDataType getTsDataType() { return iMeasurementSchema.getType(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java index c912a7c221f7..1909e1558e7b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/TimeSeriesSchemaCache.java @@ -111,6 +111,7 @@ public void computeValue(int index, SchemaCacheEntry value) { value.getIMeasurementSchema(), value.getTagMap(), null, + null, value.isAligned()); storageGroupSet.add(value.getStorageGroup()); } @@ -130,6 +131,7 @@ public ClusterSchemaTree get(PartialPath fullPath) { schemaCacheEntry.getIMeasurementSchema(), schemaCacheEntry.getTagMap(), null, + null, schemaCacheEntry.isAligned()); schemaTree.setDatabases(Collections.singleton(schemaCacheEntry.getStorageGroup())); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java index 630ca1f8cce2..3d2cae267fc2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/AutoCreateSchemaExecutor.java @@ -493,6 +493,7 @@ private void internalCreateTimeSeries( measurements.get(i), tsDataTypes.get(i), encodings.get(i), compressors.get(i)), null, null, + null, isAligned); } } @@ -620,6 +621,7 @@ private void internalCreateTimeSeries( measurementGroup.getCompressors().get(i)), null, null, + null, entry.getValue().left); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java index 84b93546cdcf..64fe751c3778 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetchExecutor.java @@ -34,7 +34,8 @@ import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult; import org.apache.iotdb.db.queryengine.plan.statement.Statement; -import org.apache.iotdb.db.queryengine.plan.statement.internal.SchemaFetchStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.DeviceSchemaFetchStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.SeriesSchemaFetchStatement; import org.apache.iotdb.db.schemaengine.template.ITemplateManager; import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.db.utils.SetThreadName; @@ -107,7 +108,9 @@ ClusterSchemaTree fetchSchemaOfFuzzyMatch( templateMap.putAll(templateManager.checkAllRelatedTemplate(pattern)); } return executeSchemaFetchQuery( - new SchemaFetchStatement(patternTree, templateMap, withTags, withTemplate), context); + new SeriesSchemaFetchStatement( + patternTree, templateMap, withTags, false, withTemplate, false), + context); } /** @@ -125,8 +128,13 @@ ClusterSchemaTree fetchSchemaOfPreciseMatchOrPreciseDeviceUsingTemplate( MPPQueryContext context) { ClusterSchemaTree schemaTree = executeSchemaFetchQuery( - new SchemaFetchStatement( - rawPatternTree, analyzeTemplate(fullPathList), false, withTemplate), + new SeriesSchemaFetchStatement( + rawPatternTree, + analyzeTemplate(fullPathList), + false, + withTemplate, + withTemplate, + withTemplate), context); if (!schemaTree.isEmpty()) { schemaCacheUpdater.accept(schemaTree); @@ -134,6 +142,23 @@ rawPatternTree, analyzeTemplate(fullPathList), false, withTemplate), return schemaTree; } + ClusterSchemaTree fetchDeviceLevelRawSchema( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return executeSchemaFetchQuery( + new DeviceSchemaFetchStatement(patternTree, authorityScope), context); + } + + ClusterSchemaTree fetchMeasurementLevelRawSchema( + PathPatternTree patternTree, MPPQueryContext context) { + Map templateMap = new HashMap<>(); + List pathPatternList = patternTree.getAllPathPatterns(); + for (PartialPath pattern : pathPatternList) { + templateMap.putAll(templateManager.checkAllRelatedTemplate(pattern)); + } + return executeSchemaFetchQuery( + new SeriesSchemaFetchStatement(patternTree, templateMap, true, true, false, true), context); + } + ClusterSchemaTree fetchSchemaOfOneDevice( PartialPath devicePath, String[] measurements, @@ -189,8 +214,13 @@ private ClusterSchemaTree fetchSchemaAndCacheResult( PathPatternTree patternTree, MPPQueryContext context) { ClusterSchemaTree schemaTree = executeSchemaFetchQuery( - new SchemaFetchStatement( - patternTree, analyzeTemplate(patternTree.getAllPathPatterns()), false, true), + new SeriesSchemaFetchStatement( + patternTree, + analyzeTemplate(patternTree.getAllPathPatterns()), + false, + false, + true, + false), context); if (!schemaTree.isEmpty()) { schemaCacheUpdater.accept(schemaTree); @@ -207,11 +237,11 @@ private Map analyzeTemplate(List pathPatternList } private ClusterSchemaTree executeSchemaFetchQuery( - SchemaFetchStatement schemaFetchStatement, MPPQueryContext context) { + Statement fetchStatement, MPPQueryContext context) { long queryId = SessionManager.getInstance().requestQueryId(); Throwable t = null; try { - ExecutionResult executionResult = executionStatement(queryId, schemaFetchStatement, context); + ExecutionResult executionResult = executionStatement(queryId, fetchStatement, context); if (executionResult.status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new RuntimeException( String.format( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java index bae7472ab8d0..11c0979a2b7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ClusterSchemaFetcher.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; @@ -155,6 +156,21 @@ public ClusterSchemaTree fetchSchema( } } + @Override + public ISchemaTree fetchRawSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + authorityScope.constructTree(); + return clusterSchemaFetchExecutor.fetchDeviceLevelRawSchema( + patternTree, authorityScope, context); + } + + @Override + public ISchemaTree fetchRawSchemaInMeasurementLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return clusterSchemaFetchExecutor.fetchMeasurementLevelRawSchema( + PathPatternTreeUtils.intersectWithFullPathPrefixTree(patternTree, authorityScope), context); + } + @Override public ClusterSchemaTree fetchSchemaWithTags( PathPatternTree patternTree, boolean withTemplate, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java index a51f8af0ab13..56a7670ad9d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/schema/ISchemaFetcher.java @@ -39,29 +39,32 @@ public interface ISchemaFetcher { /** - * Fetch all the schema of existing timeseries matched by the given patternTree + * Fetch all the schema of existing timeseries matched by the given patternTree. The result schema + * tree may contain extra measurement nodes if withTemplate is true. It recommends to use pattern + * to filter the result schema tree. * * @param patternTree used for matching the timeseries + * @param withTemplate whether to include the template in the schema * @return the matched timeseries schema organized as tree structure logically */ ISchemaTree fetchSchema( PathPatternTree patternTree, boolean withTemplate, MPPQueryContext context); /** - * TODO need to be implemented in schema engine + * Fetch all the schema by the given patternTree in device level * - *

Fetch all the schema by the given patternTree in device level + * @return schemaTree without measurement nodes + */ + ISchemaTree fetchRawSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context); + + /** + * Fetch all the schema by the given patternTree in device level * * @return schemaTree without measurement nodes */ - default ISchemaTree fetchSchemaInDeviceLevel( - PathPatternTree patternTree, MPPQueryContext context) { - ISchemaTree schemaTree = fetchSchema(patternTree, false, context); - if (schemaTree.hasLogicalViewMeasurement()) { - schemaTree.removeLogicalView(); - } - return schemaTree; - } + ISchemaTree fetchRawSchemaInMeasurementLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context); /** * Fetch all the schema with tags of existing timeseries matched by the given patternTree diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java index 4db63ed1d493..465c2e2e7237 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.exception.MemoryNotEnoughException; import org.apache.iotdb.db.queryengine.execution.driver.DataDriverContext; import org.apache.iotdb.db.queryengine.execution.fragment.DataNodeQueryContext; @@ -101,7 +102,7 @@ public List plan( context.addPipelineDriverFactory(root, context.getDriverContext(), estimatedMemorySize); instanceContext.setSourcePaths(collectSourcePaths(context)); - instanceContext.setDevicePathsToAligned(collectDevicePathsToAligned(context)); + instanceContext.setDevicePathsToContext(collectDevicePathsToContext(context)); instanceContext.setQueryDataSourceType( getQueryDataSourceType((DataDriverContext) context.getDriverContext())); @@ -194,11 +195,12 @@ private QueryDataSourceType getQueryDataSourceType(DataDriverContext dataDriverC return dataDriverContext.getQueryDataSourceType().orElse(QueryDataSourceType.SERIES_SCAN); } - private Map collectDevicePathsToAligned(LocalExecutionPlanContext context) { + private Map collectDevicePathsToContext( + LocalExecutionPlanContext context) { DataDriverContext dataDriverContext = (DataDriverContext) context.getDriverContext(); - Map deviceToAlignedMap = dataDriverContext.getDeviceIDToAligned(); - dataDriverContext.clearDeviceIDToAligned(); - return deviceToAlignedMap; + Map deviceContextMap = dataDriverContext.getDeviceIDToContext(); + dataDriverContext.clearDeviceIDToContext(); + return deviceContextMap; } private List collectSourcePaths(LocalExecutionPlanContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java index 5e614e16e28f..1af2387b0d51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanBuilder.java @@ -28,8 +28,9 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.execution.MemoryEstimationHelper; import org.apache.iotdb.db.queryengine.execution.aggregation.AccumulatorFactory; @@ -42,6 +43,7 @@ import org.apache.iotdb.db.queryengine.plan.expression.multi.FunctionExpression; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.CountSchemaMergeNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.LevelTimeSeriesCountNode; @@ -52,9 +54,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.NodePathsSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.PathsUsingTemplateScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ColumnInjectNode; @@ -1084,12 +1086,14 @@ public LogicalPlanBuilder planSchemaFetchMerge(List storageGroupList) { } @SuppressWarnings({"checkstyle:Indentation", "checkstyle:CommentsIndentation"}) - public LogicalPlanBuilder planSchemaFetchSource( + public LogicalPlanBuilder planSeriesSchemaFetchSource( List storageGroupList, PathPatternTree patternTree, Map templateMap, boolean withTags, - boolean withTemplate) { + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) { PartialPath storageGroupPath; for (String storageGroup : storageGroupList) { try { @@ -1102,13 +1106,43 @@ public LogicalPlanBuilder planSchemaFetchSource( overlappedPatternTree.appendFullPath(pathPattern); } this.root.addChild( - new SchemaFetchScanNode( + new SeriesSchemaFetchScanNode( context.getQueryId().genPlanNodeId(), storageGroupPath, overlappedPatternTree, templateMap, withTags, - withTemplate)); + withAttributes, + withTemplate, + withAliasForce)); + } catch (IllegalPathException e) { + // definitely won't happen + throw new RuntimeException(e); + } + } + return this; + } + + @SuppressWarnings({"checkstyle:Indentation", "checkstyle:CommentsIndentation"}) + public LogicalPlanBuilder planDeviceSchemaFetchSource( + List storageGroupList, PathPatternTree patternTree, PathPatternTree authorityScope) { + PartialPath storageGroupPath; + for (String storageGroup : storageGroupList) { + try { + storageGroupPath = new PartialPath(storageGroup); + PathPatternTree overlappedPatternTree = new PathPatternTree(); + for (PartialPath pathPattern : + patternTree.getOverlappedPathPatterns( + storageGroupPath.concatNode(MULTI_LEVEL_PATH_WILDCARD))) { + // pathPattern has been deduplicated, no need to deduplicate again + overlappedPatternTree.appendFullPath(pathPattern); + } + this.root.addChild( + new DeviceSchemaFetchScanNode( + context.getQueryId().genPlanNodeId(), + storageGroupPath, + overlappedPatternTree, + authorityScope)); } catch (IllegalPathException e) { // definitely won't happen throw new RuntimeException(e); @@ -1347,15 +1381,15 @@ public LogicalPlanBuilder planEndTimeColumnInject( } public LogicalPlanBuilder planDeviceRegionScan( - Map devicePathToAlignedStatus, boolean outputCount) { + Map devicePathToContextMap, boolean outputCount) { this.root = new DeviceRegionScanNode( - context.getQueryId().genPlanNodeId(), devicePathToAlignedStatus, outputCount, null); + context.getQueryId().genPlanNodeId(), devicePathToContextMap, outputCount, null); return this; } public LogicalPlanBuilder planTimeseriesRegionScan( - Map>> deviceToTimeseriesSchemaInfo, + Map>> deviceToTimeseriesSchemaInfo, boolean outputCount) { TimeseriesRegionScanNode timeseriesRegionScanNode = new TimeseriesRegionScanNode(context.getQueryId().genPlanNodeId(), outputCount, null); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java index 3bec19321d3c..f3c3c26561e6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LogicalPlanVisitor.java @@ -61,10 +61,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.DeviceSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; -import org.apache.iotdb.db.queryengine.plan.statement.internal.SchemaFetchStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.SeriesSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDevicesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountLevelTimeSeriesStatement; @@ -578,7 +579,7 @@ public PlanNode visitShowDevices( if (showDevicesStatement.hasTimeCondition()) { planBuilder = planBuilder - .planDeviceRegionScan(analysis.getDevicePathToAlignedStatus(), false) + .planDeviceRegionScan(analysis.getDevicePathToContextMap(), false) .planLimit(showDevicesStatement.getLimit()) .planOffset(showDevicesStatement.getOffset()); return planBuilder.getRoot(); @@ -624,7 +625,7 @@ public PlanNode visitCountDevices( LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); if (countDevicesStatement.hasTimeCondition()) { - planBuilder = planBuilder.planDeviceRegionScan(analysis.getDevicePathToAlignedStatus(), true); + planBuilder = planBuilder.planDeviceRegionScan(analysis.getDevicePathToContextMap(), true); return planBuilder.getRoot(); } @@ -776,19 +777,36 @@ public PlanNode visitInsertRowsOfOneDevice( } @Override - public PlanNode visitSchemaFetch( - SchemaFetchStatement schemaFetchStatement, MPPQueryContext context) { + public PlanNode visitSeriesSchemaFetch( + SeriesSchemaFetchStatement seriesSchemaFetchStatement, MPPQueryContext context) { LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); List storageGroupList = new ArrayList<>(analysis.getSchemaPartitionInfo().getSchemaPartitionMap().keySet()); return planBuilder .planSchemaFetchMerge(storageGroupList) - .planSchemaFetchSource( + .planSeriesSchemaFetchSource( storageGroupList, - schemaFetchStatement.getPatternTree(), - schemaFetchStatement.getTemplateMap(), - schemaFetchStatement.isWithTags(), - schemaFetchStatement.isWithTemplate()) + seriesSchemaFetchStatement.getPatternTree(), + seriesSchemaFetchStatement.getTemplateMap(), + seriesSchemaFetchStatement.isWithTags(), + seriesSchemaFetchStatement.isWithAttributes(), + seriesSchemaFetchStatement.isWithTemplate(), + seriesSchemaFetchStatement.isWithAliasForce()) + .getRoot(); + } + + @Override + public PlanNode visitDeviceSchemaFetch( + DeviceSchemaFetchStatement deviceSchemaFetchStatement, MPPQueryContext context) { + LogicalPlanBuilder planBuilder = new LogicalPlanBuilder(analysis, context); + List storageGroupList = + new ArrayList<>(analysis.getSchemaPartitionInfo().getSchemaPartitionMap().keySet()); + return planBuilder + .planSchemaFetchMerge(storageGroupList) + .planDeviceSchemaFetchSource( + storageGroupList, + deviceSchemaFetchStatement.getPatternTree(), + deviceSchemaFetchStatement.getAuthorityScope()) .getRoot(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java index 4f07c31b2677..693c52adbdd4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/OperatorTreeGenerator.java @@ -26,9 +26,10 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.FragmentInstanceId; import org.apache.iotdb.db.queryengine.common.NodeRef; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.execution.aggregation.Accumulator; import org.apache.iotdb.db.queryengine.execution.aggregation.AccumulatorFactory; import org.apache.iotdb.db.queryengine.execution.aggregation.Aggregator; @@ -167,6 +168,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.CountSchemaMergeNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.LevelTimeSeriesCountNode; @@ -177,10 +179,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.NodePathsSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.PathsUsingTemplateScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ActiveRegionScanMergeNode; @@ -2650,8 +2652,8 @@ public Operator visitSchemaFetchMerge( } @Override - public Operator visitSchemaFetchScan( - SchemaFetchScanNode node, LocalExecutionPlanContext context) { + public Operator visitSeriesSchemaFetchScan( + SeriesSchemaFetchScanNode node, LocalExecutionPlanContext context) { OperatorContext operatorContext = context .getDriverContext() @@ -2659,14 +2661,34 @@ public Operator visitSchemaFetchScan( context.getNextOperatorId(), node.getPlanNodeId(), SchemaFetchScanOperator.class.getSimpleName()); - return new SchemaFetchScanOperator( + return SchemaFetchScanOperator.ofSeries( node.getPlanNodeId(), operatorContext, node.getPatternTree(), node.getTemplateMap(), ((SchemaDriverContext) (context.getDriverContext())).getSchemaRegion(), node.isWithTags(), - node.isWithTemplate()); + node.isWithAttributes(), + node.isWithTemplate(), + node.isWithAliasForce()); + } + + @Override + public Operator visitDeviceSchemaFetchScan( + DeviceSchemaFetchScanNode node, LocalExecutionPlanContext context) { + OperatorContext operatorContext = + context + .getDriverContext() + .addOperatorContext( + context.getNextOperatorId(), + node.getPlanNodeId(), + SchemaFetchScanOperator.class.getSimpleName()); + return SchemaFetchScanOperator.ofDevice( + node.getPlanNodeId(), + operatorContext, + node.getPatternTree(), + node.getAuthorityScope(), + ((SchemaDriverContext) (context.getDriverContext())).getSchemaRegion()); } @Override @@ -3591,17 +3613,18 @@ public Operator visitDeviceRegionScan( node.getPlanNodeId(), ActiveDeviceRegionScanOperator.class.getSimpleName()); Filter filter = context.getGlobalTimeFilter(); - Map deviceIDToAligned = new HashMap<>(); - for (Map.Entry entry : node.getDevicePathsToAligned().entrySet()) { - deviceIDToAligned.put(new PlainDeviceID(entry.getKey().getFullPath()), entry.getValue()); + Map deviceIDToContext = new HashMap<>(); + for (Map.Entry entry : + node.getDevicePathToContextMap().entrySet()) { + deviceIDToContext.put(new PlainDeviceID(entry.getKey().getFullPath()), entry.getValue()); } ActiveDeviceRegionScanOperator regionScanOperator = new ActiveDeviceRegionScanOperator( - operatorContext, node.getPlanNodeId(), deviceIDToAligned, filter, node.isOutputCount()); + operatorContext, node.getPlanNodeId(), deviceIDToContext, filter, node.isOutputCount()); DataDriverContext dataDriverContext = (DataDriverContext) context.getDriverContext(); dataDriverContext.addSourceOperator(regionScanOperator); - dataDriverContext.setDeviceIDToAligned(deviceIDToAligned); + dataDriverContext.setDeviceIDToContext(deviceIDToContext); dataDriverContext.setQueryDataSourceType(QueryDataSourceType.DEVICE_REGION_SCAN); return regionScanOperator; @@ -3620,10 +3643,10 @@ public Operator visitTimeSeriesRegionScan( Filter filter = context.getGlobalTimeFilter(); DataDriverContext dataDriverContext = (DataDriverContext) context.getDriverContext(); - Map> timeseriesToSchemaInfo = new HashMap<>(); - for (Map.Entry>> entryMap : + Map> timeseriesToSchemaInfo = new HashMap<>(); + for (Map.Entry>> entryMap : node.getDeviceToTimeseriesSchemaInfo().entrySet()) { - Map timeseriesSchemaInfoMap = + Map timeseriesSchemaInfoMap = getTimeseriesSchemaInfoMap(entryMap, dataDriverContext); timeseriesToSchemaInfo.put( new PlainDeviceID(entryMap.getKey().getFullPath()), timeseriesSchemaInfoMap); @@ -3642,12 +3665,11 @@ public Operator visitTimeSeriesRegionScan( return regionScanOperator; } - private static Map getTimeseriesSchemaInfoMap( - Map.Entry>> entryMap, + private static Map getTimeseriesSchemaInfoMap( + Map.Entry>> entryMap, DataDriverContext context) { - Map timeseriesSchemaInfoMap = new HashMap<>(); - for (Map.Entry> entry : - entryMap.getValue().entrySet()) { + Map timeseriesSchemaInfoMap = new HashMap<>(); + for (Map.Entry> entry : entryMap.getValue().entrySet()) { PartialPath path = entry.getKey(); context.addPath(path); if (path instanceof MeasurementPath) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java index f261acfb65c2..e5983b6a18ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/ExchangeNodeAdder.java @@ -28,11 +28,12 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.AbstractSchemaMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.CountSchemaMergeNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.ActiveRegionScanMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationMergeSortNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.AggregationNode; @@ -158,7 +159,14 @@ public PlanNode visitSchemaQueryScan(SchemaQueryScanNode node, NodeGroupContext } @Override - public PlanNode visitSchemaFetchScan(SchemaFetchScanNode node, NodeGroupContext context) { + public PlanNode visitSeriesSchemaFetchScan( + SeriesSchemaFetchScanNode node, NodeGroupContext context) { + return processNoChildSourceNode(node, context); + } + + @Override + public PlanNode visitDeviceSchemaFetchScan( + DeviceSchemaFetchScanNode node, NodeGroupContext context) { return processNoChildSourceNode(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index b546516be388..c1432bf14512 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.CountSchemaMergeNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.LevelTimeSeriesCountNode; @@ -31,9 +32,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.NodePathsSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.PathsUsingTemplateScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ActivateTemplateNode; @@ -142,7 +143,7 @@ public enum PlanNodeType { ALTER_TIME_SERIES((short) 21), CREATE_ALIGNED_TIME_SERIES((short) 22), TIME_SERIES_SCHEMA_SCAN((short) 23), - SCHEMA_FETCH_SCAN((short) 24), + SERIES_SCHEMA_FETCH_SCAN((short) 24), SCHEMA_QUERY_MERGE((short) 25), SCHEMA_QUERY_ORDER_BY_HEAT((short) 26), DEVICES_COUNT((short) 27), @@ -219,7 +220,9 @@ public enum PlanNodeType { DEVICE_REGION_SCAN((short) 93), TIMESERIES_REGION_SCAN((short) 94), - REGION_MERGE((short) 95); + REGION_MERGE((short) 95), + DEVICE_SCHEMA_FETCH_SCAN((short) 96), + ; public static final int BYTES = Short.BYTES; @@ -325,7 +328,7 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { case 23: return TimeSeriesSchemaScanNode.deserialize(buffer); case 24: - return SchemaFetchScanNode.deserialize(buffer); + return SeriesSchemaFetchScanNode.deserialize(buffer); case 25: return SchemaQueryMergeNode.deserialize(buffer); case 26: @@ -462,6 +465,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return TimeseriesRegionScanNode.deserialize(buffer); case 95: return ActiveRegionScanMergeNode.deserialize(buffer); + case 96: + return DeviceSchemaFetchScanNode.deserialize(buffer); default: throw new IllegalArgumentException("Invalid node type: " + nodeType); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 81d7930dc921..2654487da54c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.CountSchemaMergeNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DevicesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.LevelTimeSeriesCountNode; @@ -27,10 +28,10 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.NodePathsCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.NodePathsSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryMergeNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryOrderByHeatNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaQueryScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesCountNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.TimeSeriesSchemaScanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.write.ActivateTemplateNode; @@ -385,7 +386,11 @@ public R visitSchemaFetchMerge(SchemaFetchMergeNode node, C context) { return visitPlan(node, context); } - public R visitSchemaFetchScan(SchemaFetchScanNode node, C context) { + public R visitSeriesSchemaFetchScan(SeriesSchemaFetchScanNode node, C context) { + return visitPlan(node, context); + } + + public R visitDeviceSchemaFetchScan(DeviceSchemaFetchScanNode node, C context) { return visitPlan(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/DeviceSchemaFetchScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/DeviceSchemaFetchScanNode.java new file mode 100644 index 000000000000..5b1935e1c4c0 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/DeviceSchemaFetchScanNode.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathDeserializeUtil; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class DeviceSchemaFetchScanNode extends SchemaFetchScanNode { + + private final PathPatternTree authorityScope; + + public DeviceSchemaFetchScanNode( + PlanNodeId id, + PartialPath storageGroup, + PathPatternTree patternTree, + PathPatternTree authorityScope) { + super(id, storageGroup, patternTree); + this.authorityScope = authorityScope; + this.authorityScope.constructTree(); + } + + public PathPatternTree getAuthorityScope() { + return authorityScope; + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.DEVICE_SCHEMA_FETCH_SCAN; + } + + @Override + public PlanNode clone() { + return new DeviceSchemaFetchScanNode( + getPlanNodeId(), storageGroup, patternTree, authorityScope); + } + + @Override + public String toString() { + return String.format( + "DeviceSchemaFetchScanNode-%s:[StorageGroup: %s, DataRegion: %s]", + this.getPlanNodeId(), + storageGroup, + PlanNodeUtil.printRegionReplicaSet(getRegionReplicaSet())); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.DEVICE_SCHEMA_FETCH_SCAN.serialize(byteBuffer); + storageGroup.serialize(byteBuffer); + patternTree.serialize(byteBuffer); + authorityScope.serialize(byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.DEVICE_SCHEMA_FETCH_SCAN.serialize(stream); + storageGroup.serialize(stream); + patternTree.serialize(stream); + authorityScope.serialize(stream); + } + + public static DeviceSchemaFetchScanNode deserialize(ByteBuffer byteBuffer) { + PartialPath storageGroup = (PartialPath) PathDeserializeUtil.deserialize(byteBuffer); + PathPatternTree patternTree = PathPatternTree.deserialize(byteBuffer); + PathPatternTree authorityScope = PathPatternTree.deserialize(byteBuffer); + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); + return new DeviceSchemaFetchScanNode(planNodeId, storageGroup, patternTree, authorityScope); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitDeviceSchemaFetchScan(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java index e68aebef1650..56d2d5cf6574 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SchemaFetchScanNode.java @@ -21,51 +21,27 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeUtil; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.source.SourceNode; -import org.apache.iotdb.db.schemaengine.template.Template; import com.google.common.collect.ImmutableList; -import org.apache.tsfile.utils.ReadWriteIOUtils; -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; - -/** This class defines the scan task of schema fetcher. */ -public class SchemaFetchScanNode extends SourceNode { - - private final PartialPath storageGroup; - private final PathPatternTree patternTree; - private final Map templateMap; - private final boolean withTags; - private final boolean withTemplate; - private TRegionReplicaSet schemaRegionReplicaSet; - - public SchemaFetchScanNode( - PlanNodeId id, - PartialPath storageGroup, - PathPatternTree patternTree, - Map templateMap, - boolean withTags, - boolean withTemplate) { + +public abstract class SchemaFetchScanNode extends SourceNode { + protected final PartialPath storageGroup; + protected final PathPatternTree patternTree; + protected TRegionReplicaSet schemaRegionReplicaSet; + + protected SchemaFetchScanNode( + PlanNodeId id, PartialPath storageGroup, PathPatternTree patternTree) { super(id); this.storageGroup = storageGroup; this.patternTree = patternTree; this.patternTree.constructTree(); - this.templateMap = templateMap; - this.withTags = withTags; - this.withTemplate = withTemplate; } public PartialPath getStorageGroup() { @@ -76,10 +52,6 @@ public PathPatternTree getPatternTree() { return patternTree; } - public Map getTemplateMap() { - return templateMap; - } - @Override public List getChildren() { return Collections.emptyList(); @@ -88,17 +60,6 @@ public List getChildren() { @Override public void addChild(PlanNode child) {} - @Override - public PlanNodeType getType() { - return PlanNodeType.SCHEMA_FETCH_SCAN; - } - - @Override - public PlanNode clone() { - return new SchemaFetchScanNode( - getPlanNodeId(), storageGroup, patternTree, templateMap, withTags, withTemplate); - } - @Override public int allowedChildCount() { return 0; @@ -109,61 +70,6 @@ public List getOutputColumnNames() { return ImmutableList.of(); } - @Override - public String toString() { - return String.format( - "SchemaFetchScanNode-%s:[StorageGroup: %s, DataRegion: %s]", - this.getPlanNodeId(), - storageGroup, - PlanNodeUtil.printRegionReplicaSet(getRegionReplicaSet())); - } - - @Override - protected void serializeAttributes(ByteBuffer byteBuffer) { - PlanNodeType.SCHEMA_FETCH_SCAN.serialize(byteBuffer); - storageGroup.serialize(byteBuffer); - patternTree.serialize(byteBuffer); - - ReadWriteIOUtils.write(templateMap.size(), byteBuffer); - for (Template template : templateMap.values()) { - template.serialize(byteBuffer); - } - ReadWriteIOUtils.write(withTags, byteBuffer); - ReadWriteIOUtils.write(withTemplate, byteBuffer); - } - - @Override - protected void serializeAttributes(DataOutputStream stream) throws IOException { - PlanNodeType.SCHEMA_FETCH_SCAN.serialize(stream); - storageGroup.serialize(stream); - patternTree.serialize(stream); - ReadWriteIOUtils.write(templateMap.size(), stream); - for (Template template : templateMap.values()) { - template.serialize(stream); - } - ReadWriteIOUtils.write(withTags, stream); - ReadWriteIOUtils.write(withTemplate, stream); - } - - public static SchemaFetchScanNode deserialize(ByteBuffer byteBuffer) { - PartialPath storageGroup = (PartialPath) PathDeserializeUtil.deserialize(byteBuffer); - PathPatternTree patternTree = PathPatternTree.deserialize(byteBuffer); - - int templateNum = ReadWriteIOUtils.readInt(byteBuffer); - Map templateMap = new HashMap<>(); - Template template; - for (int i = 0; i < templateNum; i++) { - template = new Template(); - template.deserialize(byteBuffer); - templateMap.put(template.getId(), template); - } - boolean withTags = ReadWriteIOUtils.readBool(byteBuffer); - boolean withTemplate = ReadWriteIOUtils.readBool(byteBuffer); - PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); - return new SchemaFetchScanNode( - planNodeId, storageGroup, patternTree, templateMap, withTags, withTemplate); - } - @Override public void open() throws Exception {} @@ -179,17 +85,4 @@ public void setRegionReplicaSet(TRegionReplicaSet schemaRegionReplicaSet) { @Override public void close() throws Exception {} - - @Override - public R accept(PlanVisitor visitor, C context) { - return visitor.visitSchemaFetchScan(this, context); - } - - public boolean isWithTags() { - return withTags; - } - - public boolean isWithTemplate() { - return withTemplate; - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java new file mode 100644 index 000000000000..8f3bab5e9cc1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/read/SeriesSchemaFetchScanNode.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathDeserializeUtil; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.schemaengine.template.Template; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +/** This class defines the scan task of schema fetcher. */ +public class SeriesSchemaFetchScanNode extends SchemaFetchScanNode { + + private final Map templateMap; + private final boolean withTags; + private final boolean withAttributes; + private final boolean withTemplate; + private final boolean withAliasForce; + + public SeriesSchemaFetchScanNode( + PlanNodeId id, + PartialPath storageGroup, + PathPatternTree patternTree, + Map templateMap, + boolean withTags, + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) { + super(id, storageGroup, patternTree); + this.templateMap = templateMap; + this.withTags = withTags; + this.withAttributes = withAttributes; + this.withTemplate = withTemplate; + this.withAliasForce = withAliasForce; + } + + public Map getTemplateMap() { + return templateMap; + } + + @Override + public PlanNodeType getType() { + return PlanNodeType.SERIES_SCHEMA_FETCH_SCAN; + } + + @Override + public PlanNode clone() { + return new SeriesSchemaFetchScanNode( + getPlanNodeId(), + storageGroup, + patternTree, + templateMap, + withTags, + withAttributes, + withTemplate, + withAliasForce); + } + + @Override + public String toString() { + return String.format( + "SeriesSchemaFetchScanNode-%s:[StorageGroup: %s, DataRegion: %s]", + this.getPlanNodeId(), + storageGroup, + PlanNodeUtil.printRegionReplicaSet(getRegionReplicaSet())); + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.SERIES_SCHEMA_FETCH_SCAN.serialize(byteBuffer); + storageGroup.serialize(byteBuffer); + patternTree.serialize(byteBuffer); + + ReadWriteIOUtils.write(templateMap.size(), byteBuffer); + for (Template template : templateMap.values()) { + template.serialize(byteBuffer); + } + ReadWriteIOUtils.write(withTags, byteBuffer); + ReadWriteIOUtils.write(withAttributes, byteBuffer); + ReadWriteIOUtils.write(withTemplate, byteBuffer); + ReadWriteIOUtils.write(withAliasForce, byteBuffer); + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.SERIES_SCHEMA_FETCH_SCAN.serialize(stream); + storageGroup.serialize(stream); + patternTree.serialize(stream); + ReadWriteIOUtils.write(templateMap.size(), stream); + for (Template template : templateMap.values()) { + template.serialize(stream); + } + ReadWriteIOUtils.write(withTags, stream); + ReadWriteIOUtils.write(withAttributes, stream); + ReadWriteIOUtils.write(withTemplate, stream); + ReadWriteIOUtils.write(withAliasForce, stream); + } + + public static SeriesSchemaFetchScanNode deserialize(ByteBuffer byteBuffer) { + PartialPath storageGroup = (PartialPath) PathDeserializeUtil.deserialize(byteBuffer); + PathPatternTree patternTree = PathPatternTree.deserialize(byteBuffer); + + int templateNum = ReadWriteIOUtils.readInt(byteBuffer); + Map templateMap = new HashMap<>(); + Template template; + for (int i = 0; i < templateNum; i++) { + template = new Template(); + template.deserialize(byteBuffer); + templateMap.put(template.getId(), template); + } + boolean withTags = ReadWriteIOUtils.readBool(byteBuffer); + boolean withAttributes = ReadWriteIOUtils.readBool(byteBuffer); + boolean withTemplate = ReadWriteIOUtils.readBool(byteBuffer); + boolean withAliasForce = ReadWriteIOUtils.readBool(byteBuffer); + PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); + return new SeriesSchemaFetchScanNode( + planNodeId, + storageGroup, + patternTree, + templateMap, + withTags, + withAttributes, + withTemplate, + withAliasForce); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitSeriesSchemaFetchScan(this, context); + } + + public boolean isWithTags() { + return withTags; + } + + public boolean isWithTemplate() { + return withTemplate; + } + + public boolean isWithAttributes() { + return withAttributes; + } + + public boolean isWithAliasForce() { + return withAliasForce; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/DeviceRegionScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/DeviceRegionScanNode.java index 8671c52ddc0c..a4db348342bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/DeviceRegionScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/DeviceRegionScanNode.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -45,21 +46,21 @@ import java.util.stream.Collectors; public class DeviceRegionScanNode extends RegionScanNode { - private Map devicePathsToAligned; + private Map devicePathToContextMap; public DeviceRegionScanNode( PlanNodeId planNodeId, - Map devicePathsToAligned, + Map devicePathToContextMap, boolean outputCount, TRegionReplicaSet regionReplicaSet) { super(planNodeId); - this.devicePathsToAligned = devicePathsToAligned; + this.devicePathToContextMap = devicePathToContextMap; this.regionReplicaSet = regionReplicaSet; this.outputCount = outputCount; } - public Map getDevicePathsToAligned() { - return devicePathsToAligned; + public Map getDevicePathToContextMap() { + return devicePathToContextMap; } @Override @@ -75,7 +76,7 @@ public void addChild(PlanNode child) { @Override public PlanNode clone() { return new DeviceRegionScanNode( - getPlanNodeId(), getDevicePathsToAligned(), isOutputCount(), getRegionReplicaSet()); + getPlanNodeId(), getDevicePathToContextMap(), isOutputCount(), getRegionReplicaSet()); } @Override @@ -101,24 +102,23 @@ public int allowedChildCount() { public static PlanNode deserialize(ByteBuffer buffer) { int size = ReadWriteIOUtils.readInt(buffer); - Map devicePathsToAligned = new HashMap<>(); + Map devicePathToContextMap = new HashMap<>(); for (int i = 0; i < size; i++) { PartialPath path = (PartialPath) PathDeserializeUtil.deserialize(buffer); - boolean aligned = ReadWriteIOUtils.readBool(buffer); - devicePathsToAligned.put(path, aligned); + devicePathToContextMap.put(path, DeviceContext.deserialize(buffer)); } boolean outputCount = ReadWriteIOUtils.readBool(buffer); PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); - return new DeviceRegionScanNode(planNodeId, devicePathsToAligned, outputCount, null); + return new DeviceRegionScanNode(planNodeId, devicePathToContextMap, outputCount, null); } @Override protected void serializeAttributes(ByteBuffer byteBuffer) { PlanNodeType.DEVICE_REGION_SCAN.serialize(byteBuffer); - ReadWriteIOUtils.write(devicePathsToAligned.size(), byteBuffer); - for (Map.Entry entry : devicePathsToAligned.entrySet()) { + ReadWriteIOUtils.write(devicePathToContextMap.size(), byteBuffer); + for (Map.Entry entry : devicePathToContextMap.entrySet()) { entry.getKey().serialize(byteBuffer); - ReadWriteIOUtils.write(entry.getValue(), byteBuffer); + entry.getValue().serializeAttributes(byteBuffer); } ReadWriteIOUtils.write(outputCount, byteBuffer); } @@ -126,10 +126,10 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { @Override protected void serializeAttributes(DataOutputStream stream) throws IOException { PlanNodeType.DEVICE_REGION_SCAN.serialize(stream); - ReadWriteIOUtils.write(devicePathsToAligned.size(), stream); - for (Map.Entry entry : devicePathsToAligned.entrySet()) { + ReadWriteIOUtils.write(devicePathToContextMap.size(), stream); + for (Map.Entry entry : devicePathToContextMap.entrySet()) { entry.getKey().serialize(stream); - ReadWriteIOUtils.write(entry.getValue(), stream); + entry.getValue().serializeAttributes(stream); } ReadWriteIOUtils.write(outputCount, stream); } @@ -145,23 +145,23 @@ public String toString() { @Override public Set getDevicePaths() { - return new HashSet<>(devicePathsToAligned.keySet()); + return new HashSet<>(devicePathToContextMap.keySet()); } @Override public void addDevicePath(PartialPath devicePath, RegionScanNode node) { - this.devicePathsToAligned.put( - devicePath, ((DeviceRegionScanNode) node).devicePathsToAligned.get(devicePath)); + this.devicePathToContextMap.put( + devicePath, ((DeviceRegionScanNode) node).devicePathToContextMap.get(devicePath)); } @Override public void clearPath() { - this.devicePathsToAligned = new HashMap<>(); + this.devicePathToContextMap = new HashMap<>(); } @Override public long getSize() { - return devicePathsToAligned.size(); + return devicePathToContextMap.size(); } @Override @@ -170,12 +170,12 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; if (!super.equals(o)) return false; DeviceRegionScanNode that = (DeviceRegionScanNode) o; - return devicePathsToAligned.equals(that.devicePathsToAligned) + return devicePathToContextMap.equals(that.devicePathToContextMap) && outputCount == that.isOutputCount(); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), devicePathsToAligned, outputCount); + return Objects.hash(super.hashCode(), devicePathToContextMap, outputCount); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java index 2bbe27d2230d..5a5abed1aa02 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/source/TimeseriesRegionScanNode.java @@ -26,7 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathType; import org.apache.iotdb.commons.utils.TestOnly; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; @@ -55,8 +55,7 @@ public class TimeseriesRegionScanNode extends RegionScanNode { // IDeviceID -> (MeasurementPath -> TimeseriesSchemaInfo) - private Map>> - deviceToTimeseriesSchemaInfo; + private Map>> deviceToTimeseriesSchemaInfo; public TimeseriesRegionScanNode( PlanNodeId planNodeId, boolean outputCount, TRegionReplicaSet regionReplicaSet) { @@ -67,7 +66,7 @@ public TimeseriesRegionScanNode( public TimeseriesRegionScanNode( PlanNodeId planNodeId, - Map>> deviceToTimeseriesSchemaInfo, + Map>> deviceToTimeseriesSchemaInfo, boolean outputCount, TRegionReplicaSet regionReplicaSet) { super(planNodeId); @@ -77,11 +76,11 @@ public TimeseriesRegionScanNode( } public void setDeviceToTimeseriesSchemaInfo( - Map>> deviceToTimeseriesSchemaInfo) { + Map>> deviceToTimeseriesSchemaInfo) { this.deviceToTimeseriesSchemaInfo = deviceToTimeseriesSchemaInfo; } - public Map>> + public Map>> getDeviceToTimeseriesSchemaInfo() { return deviceToTimeseriesSchemaInfo; } @@ -125,7 +124,7 @@ public int allowedChildCount() { public static PlanNode deserialize(ByteBuffer buffer) { int size = ReadWriteIOUtils.readInt(buffer); - Map>> deviceToTimeseriesSchemaInfo = + Map>> deviceToTimeseriesSchemaInfo = new HashMap<>(); for (int i = 0; i < size; i++) { @@ -137,13 +136,13 @@ public static PlanNode deserialize(ByteBuffer buffer) { PartialPath devicePath = new PartialPath(nodes); int pathSize = ReadWriteIOUtils.readInt(buffer); - Map> measurementToSchemaInfo = new HashMap<>(); + Map> measurementToSchemaInfo = new HashMap<>(); for (int j = 0; j < pathSize; j++) { PartialPath path = deserializePartialPath(nodes, buffer); int schemaSize = ReadWriteIOUtils.readInt(buffer); - List schemaInfos = new ArrayList<>(); + List schemaInfos = new ArrayList<>(); for (int k = 0; k < schemaSize; k++) { - schemaInfos.add(TimeseriesSchemaInfo.deserialize(buffer)); + schemaInfos.add(TimeseriesContext.deserialize(buffer)); } measurementToSchemaInfo.put(path, schemaInfos); } @@ -197,7 +196,7 @@ public String toString() { "%s -> %s", entry1.getKey().getFullPath(), entry1.getValue().stream() - .map(TimeseriesSchemaInfo::toString) + .map(TimeseriesContext::toString) .collect(Collectors.joining(", ")))) .collect(Collectors.joining(", ")))) .collect(Collectors.joining(", "))); @@ -247,7 +246,7 @@ public int hashCode() { protected void serializeAttributes(ByteBuffer byteBuffer) { PlanNodeType.TIMESERIES_REGION_SCAN.serialize(byteBuffer); ReadWriteIOUtils.write(deviceToTimeseriesSchemaInfo.size(), byteBuffer); - for (Map.Entry>> entry : + for (Map.Entry>> entry : deviceToTimeseriesSchemaInfo.entrySet()) { int size = entry.getKey().getNodeLength(); @@ -258,12 +257,12 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { } ReadWriteIOUtils.write(entry.getValue().size(), byteBuffer); - for (Map.Entry> timseriesEntry : + for (Map.Entry> timseriesEntry : entry.getValue().entrySet()) { serializeMeasurements(timseriesEntry.getKey(), byteBuffer); ReadWriteIOUtils.write(timseriesEntry.getValue().size(), byteBuffer); - for (TimeseriesSchemaInfo timeseriesSchemaInfo : timseriesEntry.getValue()) { - timeseriesSchemaInfo.serializeAttributes(byteBuffer); + for (TimeseriesContext timeseriesContext : timseriesEntry.getValue()) { + timeseriesContext.serializeAttributes(byteBuffer); } } } @@ -274,7 +273,7 @@ protected void serializeAttributes(ByteBuffer byteBuffer) { protected void serializeAttributes(DataOutputStream stream) throws IOException { PlanNodeType.TIMESERIES_REGION_SCAN.serialize(stream); ReadWriteIOUtils.write(deviceToTimeseriesSchemaInfo.size(), stream); - for (Map.Entry>> entry : + for (Map.Entry>> entry : deviceToTimeseriesSchemaInfo.entrySet()) { int size = entry.getKey().getNodeLength(); @@ -285,12 +284,12 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { } ReadWriteIOUtils.write(entry.getValue().size(), stream); - for (Map.Entry> timseriesEntry : + for (Map.Entry> timseriesEntry : entry.getValue().entrySet()) { serializeMeasurements(timseriesEntry.getKey(), stream); ReadWriteIOUtils.write(timseriesEntry.getValue().size(), stream); - for (TimeseriesSchemaInfo timeseriesSchemaInfo : timseriesEntry.getValue()) { - timeseriesSchemaInfo.serializeAttributes(stream); + for (TimeseriesContext timeseriesContext : timseriesEntry.getValue()) { + timeseriesContext.serializeAttributes(stream); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index d8a0a42016d0..1fd8e8a58c28 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -28,10 +28,11 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.queryengine.plan.statement.crud.QueryStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.DeviceSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalBatchActivateTemplateStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateMultiTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.internal.InternalCreateTimeSeriesStatement; -import org.apache.iotdb.db.queryengine.plan.statement.internal.SchemaFetchStatement; +import org.apache.iotdb.db.queryengine.plan.statement.internal.SeriesSchemaFetchStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.AlterTimeSeriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDatabaseStatement; import org.apache.iotdb.db.queryengine.plan.statement.metadata.CountDevicesStatement; @@ -361,8 +362,14 @@ public R visitCountNodes(CountNodesStatement countStatement, C context) { return visitStatement(countStatement, context); } - public R visitSchemaFetch(SchemaFetchStatement schemaFetchStatement, C context) { - return visitStatement(schemaFetchStatement, context); + public R visitSeriesSchemaFetch( + SeriesSchemaFetchStatement seriesSchemaFetchStatement, C context) { + return visitStatement(seriesSchemaFetchStatement, context); + } + + public R visitDeviceSchemaFetch( + DeviceSchemaFetchStatement deviceSchemaFetchStatement, C context) { + return visitStatement(deviceSchemaFetchStatement, context); } public R visitShowChildPaths(ShowChildPathsStatement showChildPathsStatement, C context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/DeviceSchemaFetchStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/DeviceSchemaFetchStatement.java new file mode 100644 index 000000000000..00271f190978 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/DeviceSchemaFetchStatement.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.internal; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.List; + +public class DeviceSchemaFetchStatement extends Statement { + private final PathPatternTree patternTree; + private final PathPatternTree authorityScope; + + public DeviceSchemaFetchStatement(PathPatternTree patternTree, PathPatternTree authorityScope) { + super(); + this.patternTree = patternTree; + this.authorityScope = authorityScope; + setType(StatementType.FETCH_SCHEMA); + } + + public PathPatternTree getPatternTree() { + return patternTree; + } + + public PathPatternTree getAuthorityScope() { + return authorityScope; + } + + @Override + public List getPaths() { + return patternTree.getAllPathPatterns(); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitDeviceSchemaFetch(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SchemaFetchStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java similarity index 80% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SchemaFetchStatement.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java index f9b3047e5700..f0b393c817f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SchemaFetchStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/internal/SeriesSchemaFetchStatement.java @@ -29,23 +29,29 @@ import java.util.List; import java.util.Map; -public class SchemaFetchStatement extends Statement { +public class SeriesSchemaFetchStatement extends Statement { private final PathPatternTree patternTree; private final Map templateMap; private final boolean withTags; + private final boolean withAttributes; private final boolean withTemplate; + private final boolean withAliasForce; - public SchemaFetchStatement( + public SeriesSchemaFetchStatement( PathPatternTree patternTree, Map templateMap, boolean withTags, - boolean withTemplate) { + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) { super(); this.patternTree = patternTree; this.templateMap = templateMap; this.withTags = withTags; + this.withAttributes = withAttributes; this.withTemplate = withTemplate; + this.withAliasForce = withAliasForce; setType(StatementType.FETCH_SCHEMA); } @@ -59,7 +65,7 @@ public Map getTemplateMap() { @Override public R accept(StatementVisitor visitor, C context) { - return visitor.visitSchemaFetch(this, context); + return visitor.visitSeriesSchemaFetch(this, context); } @Override @@ -74,4 +80,12 @@ public boolean isWithTags() { public boolean isWithTemplate() { return withTemplate; } + + public boolean isWithAttributes() { + return withAttributes; + } + + public boolean isWithAliasForce() { + return withAliasForce; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDevicesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDevicesStatement.java index 06d32f7c9b53..9b73e1f9b0a5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDevicesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountDevicesStatement.java @@ -19,9 +19,15 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTreeUtils; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; +import org.apache.iotdb.rpc.TSStatusCode; public class CountDevicesStatement extends CountStatement { @@ -43,6 +49,27 @@ public boolean hasTimeCondition() { return timeCondition != null; } + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + if (hasTimeCondition()) { + try { + if (!AuthorityChecker.SUPER_USER.equals(userName)) { + this.authorityScope = + PathPatternTreeUtils.intersectWithFullPathPrefixTree( + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_SCHEMA.ordinal()), + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_DATA.ordinal())); + } + } catch (AuthException e) { + return new TSStatus(e.getCode().getStatusCode()); + } + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } else { + return super.checkPermissionBeforeProcess(userName); + } + } + @Override public R accept(StatementVisitor visitor, C context) { return visitor.visitCountDevices(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java index c7c259030a9c..c0910185421b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/CountTimeSeriesStatement.java @@ -19,10 +19,16 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; +import org.apache.iotdb.rpc.TSStatusCode; public class CountTimeSeriesStatement extends CountStatement { @@ -54,6 +60,27 @@ public WhereCondition getTimeCondition() { return timeCondition; } + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + if (hasTimeCondition()) { + try { + if (!AuthorityChecker.SUPER_USER.equals(userName)) { + this.authorityScope = + PathPatternTreeUtils.intersectWithFullPathPrefixTree( + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_SCHEMA.ordinal()), + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_DATA.ordinal())); + } + } catch (AuthException e) { + return new TSStatus(e.getCode().getStatusCode()); + } + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } else { + return super.checkPermissionBeforeProcess(userName); + } + } + @Override public R accept(StatementVisitor visitor, C context) { return visitor.visitCountTimeSeries(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDevicesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDevicesStatement.java index 75f33c7b4a1c..b0b7f8bba317 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDevicesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDevicesStatement.java @@ -19,10 +19,16 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; +import org.apache.iotdb.rpc.TSStatusCode; import java.util.Collections; import java.util.List; @@ -78,6 +84,27 @@ public boolean hasTimeCondition() { return timeCondition != null; } + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + if (hasTimeCondition()) { + try { + if (!AuthorityChecker.SUPER_USER.equals(userName)) { + this.authorityScope = + PathPatternTreeUtils.intersectWithFullPathPrefixTree( + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_SCHEMA.ordinal()), + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_DATA.ordinal())); + } + } catch (AuthException e) { + return new TSStatus(e.getCode().getStatusCode()); + } + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } else { + return super.checkPermissionBeforeProcess(userName); + } + } + @Override public R accept(StatementVisitor visitor, C context) { return visitor.visitShowDevices(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTimeSeriesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTimeSeriesStatement.java index 82a389d62249..7d615cac3364 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTimeSeriesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowTimeSeriesStatement.java @@ -19,10 +19,16 @@ package org.apache.iotdb.db.queryengine.plan.statement.metadata; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.auth.AuthException; +import org.apache.iotdb.commons.auth.entity.PrivilegeType; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTreeUtils; import org.apache.iotdb.commons.schema.filter.SchemaFilter; +import org.apache.iotdb.db.auth.AuthorityChecker; import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; +import org.apache.iotdb.rpc.TSStatusCode; import java.util.Collections; import java.util.List; @@ -77,6 +83,27 @@ public boolean hasTimeCondition() { return timeCondition != null; } + @Override + public TSStatus checkPermissionBeforeProcess(String userName) { + if (hasTimeCondition()) { + try { + if (!AuthorityChecker.SUPER_USER.equals(userName)) { + this.authorityScope = + PathPatternTreeUtils.intersectWithFullPathPrefixTree( + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_SCHEMA.ordinal()), + AuthorityChecker.getAuthorizedPathTree( + userName, PrivilegeType.READ_DATA.ordinal())); + } + } catch (AuthException e) { + return new TSStatus(e.getCode().getStatusCode()); + } + return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + } else { + return super.checkPermissionBeforeProcess(userName); + } + } + @Override public List getPaths() { return Collections.singletonList(pathPattern); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 95e3524d3215..61ef64c1d263 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -198,11 +198,24 @@ void checkSchemaQuota(PartialPath devicePath, int timeSeriesNum) MeasurementPath fetchMeasurementPath(PartialPath fullPath) throws MetadataException; - ClusterSchemaTree fetchSchema( + ClusterSchemaTree fetchSeriesSchema( PathPatternTree patternTree, Map templateMap, boolean withTags, - boolean withTemplate) + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) + throws MetadataException; + + /** + * Fetch all the schema by the given patternTree in device level. Currently, devices with + * isAligned!=null will be filtered out. + * + * @param patternTree devices' pattern + * @param authorityScope the scope of the authority + * @return pattern tree with all leaves as device nodes + */ + ClusterSchemaTree fetchDeviceSchema(PathPatternTree patternTree, PathPatternTree authorityScope) throws MetadataException; // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index d5b7f58cfd4b..e9173cc5888a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -214,6 +214,7 @@ public synchronized void init() throws MetadataException { new MTreeBelowSGMemoryImpl( new PartialPath(storageGroupFullPath), tagManager::readTags, + tagManager::readAttributes, regionStatistics, metric); @@ -510,7 +511,8 @@ public void loadSnapshot(File latestSnapshotRootDir) { regionStatistics.activateTemplate(deviceMNode.getSchemaTemplateId()); } }, - tagManager::readTags); + tagManager::readTags, + tagManager::readAttributes); logger.info( "MTree snapshot loading of schemaRegion {} costs {}ms.", schemaRegionId, @@ -949,23 +951,34 @@ public MeasurementPath fetchMeasurementPath(PartialPath fullPath) throws Metadat } @Override - public ClusterSchemaTree fetchSchema( + public ClusterSchemaTree fetchSeriesSchema( PathPatternTree patternTree, Map templateMap, boolean withTags, - boolean withTemplate) + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) throws MetadataException { if (patternTree.isContainWildcard()) { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); for (PartialPath path : patternTree.getAllPathPatterns()) { - schemaTree.mergeSchemaTree(mtree.fetchSchema(path, templateMap, withTags, withTemplate)); + schemaTree.mergeSchemaTree( + mtree.fetchSchema( + path, templateMap, withTags, withAttributes, withTemplate, withAliasForce)); } return schemaTree; } else { - return mtree.fetchSchemaWithoutWildcard(patternTree, templateMap, withTags, withTemplate); + return mtree.fetchSchemaWithoutWildcard( + patternTree, templateMap, withTags, withAttributes, withTemplate); } } + @Override + public ClusterSchemaTree fetchDeviceSchema( + PathPatternTree patternTree, PathPatternTree authorityScope) throws MetadataException { + return mtree.fetchDeviceSchema(patternTree, authorityScope); + } + // endregion // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index ad3ee7a223bd..4dbfaf5c3bf6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -211,6 +211,7 @@ public synchronized void init() throws MetadataException { new MTreeBelowSGCachedImpl( new PartialPath(storageGroupFullPath), tagManager::readTags, + tagManager::readAttributes, this::flushCallback, measurementInitProcess(), deviceInitProcess(), @@ -549,6 +550,7 @@ public void loadSnapshot(File latestSnapshotRootDir) { measurementInitProcess(), deviceInitProcess(), tagManager::readTags, + tagManager::readAttributes, this::flushCallback); logger.info( "MTree snapshot loading of schemaRegion {} costs {}ms.", @@ -1027,23 +1029,34 @@ public MeasurementPath fetchMeasurementPath(PartialPath fullPath) throws Metadat } @Override - public ClusterSchemaTree fetchSchema( + public ClusterSchemaTree fetchSeriesSchema( PathPatternTree patternTree, Map templateMap, boolean withTags, - boolean withTemplate) + boolean withAttributes, + boolean withTemplate, + boolean withAliasForcete) throws MetadataException { if (patternTree.isContainWildcard()) { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); for (PartialPath path : patternTree.getAllPathPatterns()) { - schemaTree.mergeSchemaTree(mtree.fetchSchema(path, templateMap, withTags, withTemplate)); + schemaTree.mergeSchemaTree( + mtree.fetchSchema( + path, templateMap, withTags, withAttributes, withTemplate, withAliasForcete)); } return schemaTree; } else { - return mtree.fetchSchemaWithoutWildcard(patternTree, templateMap, withTags, withTemplate); + return mtree.fetchSchemaWithoutWildcard( + patternTree, templateMap, withTags, withAttributes, withTemplate); } } + @Override + public ClusterSchemaTree fetchDeviceSchema( + PathPatternTree patternTree, PathPatternTree authorityScope) throws MetadataException { + return mtree.fetchDeviceSchema(patternTree, authorityScope); + } + // endregion // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index e76c3931d6b4..c1a977d0300d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -122,6 +122,7 @@ public class MTreeBelowSGMemoryImpl { private final IMemMNode rootNode; private final Function, Map> tagGetter; + private final Function, Map> attributeGetter; private final IMNodeFactory nodeFactory = MNodeFactoryLoader.getInstance().getMemMNodeIMNodeFactory(); private final int levelOfSG; @@ -131,6 +132,7 @@ public class MTreeBelowSGMemoryImpl { public MTreeBelowSGMemoryImpl( PartialPath storageGroupPath, Function, Map> tagGetter, + Function, Map> attributeGetter, MemSchemaRegionStatistics regionStatistics, SchemaRegionMemMetric metric) { store = new MemMTreeStore(storageGroupPath, regionStatistics, metric); @@ -139,12 +141,14 @@ public MTreeBelowSGMemoryImpl( this.rootNode = store.generatePrefix(storageGroupPath); levelOfSG = storageGroupPath.getNodeLength() - 1; this.tagGetter = tagGetter; + this.attributeGetter = attributeGetter; } private MTreeBelowSGMemoryImpl( PartialPath storageGroupPath, MemMTreeStore store, Function, Map> tagGetter, + Function, Map> attributeGetter, MemSchemaRegionStatistics regionStatistics) { this.store = store; this.regionStatistics = regionStatistics; @@ -152,6 +156,7 @@ private MTreeBelowSGMemoryImpl( this.rootNode = store.generatePrefix(storageGroupPath); levelOfSG = storageGroupPath.getNodeLength() - 1; this.tagGetter = tagGetter; + this.attributeGetter = attributeGetter; } public void clear() { @@ -170,13 +175,15 @@ public static MTreeBelowSGMemoryImpl loadFromSnapshot( SchemaRegionMemMetric metric, Consumer> measurementProcess, Consumer> deviceProcess, - Function, Map> tagGetter) + Function, Map> tagGetter, + Function, Map> attributeGetter) throws IOException, IllegalPathException { return new MTreeBelowSGMemoryImpl( new PartialPath(storageGroupFullPath), MemMTreeStore.loadFromSnapshot( snapshotDir, measurementProcess, deviceProcess, regionStatistics, metric), tagGetter, + attributeGetter, regionStatistics); } @@ -695,7 +702,9 @@ public ClusterSchemaTree fetchSchema( PartialPath pathPattern, Map templateMap, boolean withTags, - boolean withTemplate) + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) throws MetadataException { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); try (MeasurementCollector collector = @@ -710,14 +719,15 @@ protected Void collectMeasurement(IMeasurementMNode node) { skipTemplateChildren(deviceMNode); } else { MeasurementPath path = getCurrentMeasurementPathInTraverse(node); - if (nodes[nodes.length - 1].equals(node.getAlias())) { - // only when user query with alias, the alias in path will be set - path.setMeasurementAlias(node.getAlias()); - } - if (withTags) { - path.setTagMap(tagGetter.apply(node)); - } - schemaTree.appendSingleMeasurementPath(path); + schemaTree.appendSingleMeasurement( + path, + path.getMeasurementSchema(), + withTags ? tagGetter.apply(node) : null, + withAttributes ? attributeGetter.apply(node) : null, + withAliasForce || nodes[nodes.length - 1].equals(node.getAlias()) + ? node.getAlias() + : null, + deviceMNode.isAligned()); } return null; } @@ -733,6 +743,7 @@ public ClusterSchemaTree fetchSchemaWithoutWildcard( PathPatternTree patternTree, Map templateMap, boolean withTags, + boolean withAttributes, boolean withTemplate) throws MetadataException { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); @@ -748,11 +759,13 @@ protected Void collectMeasurement(IMeasurementMNode node) { skipTemplateChildren(deviceMNode); } else { MeasurementPath path = getCurrentMeasurementPathInTraverse(node); - path.setMeasurementAlias(node.getAlias()); - if (withTags) { - path.setTagMap(tagGetter.apply(node)); - } - schemaTree.appendSingleMeasurementPath(path); + schemaTree.appendSingleMeasurement( + path, + path.getMeasurementSchema(), + withTags ? tagGetter.apply(node) : null, + withAttributes ? attributeGetter.apply(node) : null, + node.getAlias(), + deviceMNode.isAligned()); } return null; } @@ -764,6 +777,27 @@ protected Void collectMeasurement(IMeasurementMNode node) { return schemaTree; } + public ClusterSchemaTree fetchDeviceSchema( + PathPatternTree patternTree, PathPatternTree authorityScope) throws MetadataException { + ClusterSchemaTree schemaTree = new ClusterSchemaTree(); + for (PartialPath pattern : patternTree.getAllPathPatterns()) { + try (EntityCollector collector = + new EntityCollector(rootNode, pattern, store, false, authorityScope) { + @Override + protected Void collectEntity(IDeviceMNode node) { + if (node.isAlignedNullable() != null) { + schemaTree.appendTemplateDevice( + node.getPartialPath(), node.isAligned(), node.getSchemaTemplateId(), null); + } + return null; + } + }) { + collector.traverse(); + } + } + return schemaTree; + } + // endregion // region Interfaces and Implementation for MNode Query diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java index d2ef703ee7f2..12af832364c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java @@ -121,6 +121,7 @@ public class MTreeBelowSGCachedImpl { private final ICachedMNode rootNode; private final Function, Map> tagGetter; + private final Function, Map> attributeGetter; private final IMNodeFactory nodeFactory = MNodeFactoryLoader.getInstance().getCachedMNodeIMNodeFactory(); private final int levelOfSG; @@ -130,6 +131,7 @@ public class MTreeBelowSGCachedImpl { public MTreeBelowSGCachedImpl( PartialPath storageGroupPath, Function, Map> tagGetter, + Function, Map> attributeGetter, Runnable flushCallback, Consumer> measurementProcess, Consumer> deviceProcess, @@ -138,6 +140,7 @@ public MTreeBelowSGCachedImpl( SchemaRegionCachedMetric metric) throws MetadataException, IOException { this.tagGetter = tagGetter; + this.attributeGetter = attributeGetter; this.regionStatistics = regionStatistics; store = PBTreeFactory.getInstance() @@ -177,6 +180,7 @@ private MTreeBelowSGCachedImpl( Consumer> measurementProcess, Consumer> deviceProcess, Function, Map> tagGetter, + Function, Map> attributeGetter, CachedSchemaRegionStatistics regionStatistics) throws MetadataException { this.store = store; @@ -185,6 +189,7 @@ private MTreeBelowSGCachedImpl( this.rootNode = store.generatePrefix(storageGroupPath); levelOfSG = storageGroupMNode.getPartialPath().getNodeLength() - 1; this.tagGetter = tagGetter; + this.attributeGetter = attributeGetter; // recover MNode try (MNodeCollector collector = @@ -226,6 +231,7 @@ public static MTreeBelowSGCachedImpl loadFromSnapshot( Consumer> measurementProcess, Consumer> deviceProcess, Function, Map> tagGetter, + Function, Map> attributeGetter, Runnable flushCallback) throws IOException, MetadataException { return new MTreeBelowSGCachedImpl( @@ -241,6 +247,7 @@ public static MTreeBelowSGCachedImpl loadFromSnapshot( measurementProcess, deviceProcess, tagGetter, + attributeGetter, regionStatistics); } @@ -818,7 +825,9 @@ public ClusterSchemaTree fetchSchema( PartialPath pathPattern, Map templateMap, boolean withTags, - boolean withTemplate) + boolean withAttributes, + boolean withTemplate, + boolean withAliasForce) throws MetadataException { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); try (MeasurementCollector collector = @@ -834,14 +843,15 @@ protected Void collectMeasurement(IMeasurementMNode node) { skipTemplateChildren(deviceMNode); } else { MeasurementPath path = getCurrentMeasurementPathInTraverse(node); - if (nodes[nodes.length - 1].equals(node.getAlias())) { - // only when user query with alias, the alias in path will be set - path.setMeasurementAlias(node.getAlias()); - } - if (withTags) { - path.setTagMap(tagGetter.apply(node)); - } - schemaTree.appendSingleMeasurementPath(path); + schemaTree.appendSingleMeasurement( + path, + path.getMeasurementSchema(), + withTags ? tagGetter.apply(node) : null, + withAttributes ? attributeGetter.apply(node) : null, + withAliasForce || nodes[nodes.length - 1].equals(node.getAlias()) + ? node.getAlias() + : null, + deviceMNode.isAligned()); } return null; } @@ -857,6 +867,7 @@ public ClusterSchemaTree fetchSchemaWithoutWildcard( PathPatternTree patternTree, Map templateMap, boolean withTags, + boolean withAttributes, boolean withTemplate) throws MetadataException { ClusterSchemaTree schemaTree = new ClusterSchemaTree(); @@ -873,11 +884,13 @@ protected Void collectMeasurement(IMeasurementMNode node) { skipTemplateChildren(deviceMNode); } else { MeasurementPath path = getCurrentMeasurementPathInTraverse(node); - path.setMeasurementAlias(node.getAlias()); - if (withTags) { - path.setTagMap(tagGetter.apply(node)); - } - schemaTree.appendSingleMeasurementPath(path); + schemaTree.appendSingleMeasurement( + path, + path.getMeasurementSchema(), + withTags ? tagGetter.apply(node) : null, + withAttributes ? attributeGetter.apply(node) : null, + node.getAlias(), + deviceMNode.isAligned()); } return null; } @@ -889,6 +902,27 @@ protected Void collectMeasurement(IMeasurementMNode node) { return schemaTree; } + public ClusterSchemaTree fetchDeviceSchema( + PathPatternTree patternTree, PathPatternTree authorityScope) throws MetadataException { + ClusterSchemaTree schemaTree = new ClusterSchemaTree(); + for (PartialPath pattern : patternTree.getAllPathPatterns()) { + try (EntityCollector collector = + new EntityCollector(rootNode, pattern, store, false, authorityScope) { + @Override + protected Void collectEntity(IDeviceMNode node) { + if (node.isAlignedNullable() != null) { + schemaTree.appendTemplateDevice( + node.getPartialPath(), node.isAligned(), node.getSchemaTemplateId(), null); + } + return null; + } + }) { + collector.traverse(); + } + } + return schemaTree; + } + // endregion // region Interfaces and Implementation for MNode Query diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java index 2643635731b3..d184c4647f1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/tag/TagManager.java @@ -739,6 +739,21 @@ public Map readTags(IMeasurementMNode node) { } } + /** + * Read the attributes of this node. + * + * @param node the node to query. + * @return the attribute key-value map. + * @throws RuntimeException If any IOException happens. + */ + public Map readAttributes(IMeasurementMNode node) { + try { + return readTagFile(node.getOffset()).getRight(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public void clear() throws IOException { this.tagIndex.clear(); if (tagLogFile != null) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 5a58b3e1dbfc..8c473652c939 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -46,6 +46,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.exception.quota.ExceedQuotaException; import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.execution.load.LoadTsFileRateLimiter; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; @@ -1871,7 +1872,7 @@ private List getFileHandleListForQuery( @Override public IQueryDataSource queryForDeviceRegionScan( - Map devicePathToAligned, + Map devicePathToAligned, QueryContext queryContext, Filter globalTimeFilter, List timePartitions) @@ -1906,7 +1907,7 @@ public IQueryDataSource queryForDeviceRegionScan( private List getFileHandleListForQuery( Collection tsFileResources, - Map devicePathToAligned, + Map devicePathsToContext, QueryContext context, Filter globalTimeFilter, boolean isSeq) @@ -1924,7 +1925,7 @@ private List getFileHandleListForQuery( } else { tsFileResource .getProcessor() - .queryForDeviceRegionScan(devicePathToAligned, context, fileScanHandles); + .queryForDeviceRegionScan(devicePathsToContext, context, fileScanHandles); } } finally { closeQueryLock.readLock().unlock(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IDataRegionForQuery.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IDataRegionForQuery.java index 1ad4de9abf85..c18262145e53 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IDataRegionForQuery.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/IDataRegionForQuery.java @@ -20,6 +20,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.storageengine.dataregion.read.IQueryDataSource; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; @@ -49,7 +50,7 @@ QueryDataSource query( /** Get satisfied QueryDataSource from DataRegion for regionScan */ IQueryDataSource queryForDeviceRegionScan( - Map devicePathToAligned, + Map devicePathsToContext, QueryContext queryContext, Filter globalTimeFilter, List timePartitions) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java index b18dbb01eb7e..42c5a766886c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/VirtualDataRegion.java @@ -20,6 +20,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.storageengine.dataregion.read.IQueryDataSource; import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; @@ -73,7 +74,7 @@ public QueryDataSource query( @Override public IQueryDataSource queryForDeviceRegionScan( - Map devicePathToAligned, + Map devicePathsToContext, QueryContext queryContext, Filter globalTimeFilter, List timePartitions) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 74aed59ab887..5a964c33600d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -37,6 +37,7 @@ import org.apache.iotdb.db.exception.query.QueryProcessException; import org.apache.iotdb.db.pipe.agent.PipeAgent; import org.apache.iotdb.db.pipe.extractor.dataregion.realtime.listener.PipeInsertionDataNodeListener; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet; import org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet; @@ -1883,7 +1884,7 @@ public void queryForSeriesRegionScan( * get the related ChunkMetadata of data on disk. */ public void queryForDeviceRegionScan( - Map devicePathToAligned, + Map devicePathsToContext, QueryContext queryContext, List fileScanHandlesForQuery) { long startTime = System.nanoTime(); @@ -1893,9 +1894,9 @@ public void queryForDeviceRegionScan( new HashMap<>(); flushQueryLock.readLock().lock(); try { - for (Map.Entry entry : devicePathToAligned.entrySet()) { + for (Map.Entry entry : devicePathsToContext.entrySet()) { IDeviceID deviceID = entry.getKey(); - boolean isAligned = entry.getValue(); + boolean isAligned = entry.getValue().isAligned(); long timeLowerBound = getQueryTimeLowerBound( PathUtils.splitPathToDetachedNodes(((PlainDeviceID) deviceID).toStringID())); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java index f264bd4ba3b1..fc1e2f6ff694 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/cache/DataNodeSchemaCacheTest.java @@ -206,18 +206,21 @@ private ISchemaTree generateSchemaTree1() throws IllegalPathException { new MeasurementSchema("s1", TSDataType.INT32), s1TagMap, null, + null, false); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s2"), new MeasurementSchema("s2", TSDataType.FLOAT), null, null, + null, false); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s3"), new MeasurementSchema("s3", TSDataType.BOOLEAN), null, null, + null, false); schemaTree.setDatabases(Collections.singleton("root.sg1")); return schemaTree; @@ -231,18 +234,21 @@ private ISchemaTree generateSchemaTree2() throws IllegalPathException { new MeasurementSchema("s3", TSDataType.BOOLEAN), null, null, + null, false); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s4"), new MeasurementSchema("s4", TSDataType.TEXT), null, null, + null, false); schemaTree.appendSingleMeasurement( new PartialPath("root.sg1.d1.s5"), new MeasurementSchema("s5", TSDataType.INT64), null, null, + null, false); schemaTree.setDatabases(Collections.singleton("root.sg1")); return schemaTree; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java index 4064376cd96f..6dcb9200c059 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java @@ -103,11 +103,11 @@ public void testFetchSchemaPerfomance() throws Exception { } } patternTree.constructTree(); - schemaRegion.fetchSchema(patternTree, Collections.EMPTY_MAP, false, true); + schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, false, false, true, false); long startTime; startTime = System.currentTimeMillis(); for (int i = 0; i < 10; i++) { - schemaRegion.fetchSchema(patternTree, Collections.EMPTY_MAP, false, true); + schemaRegion.fetchSeriesSchema(patternTree, Collections.EMPTY_MAP, false, false, true, false); } System.out.println("cost time: " + (System.currentTimeMillis() - startTime)); } @@ -153,7 +153,8 @@ public void testFetchSchema() throws Exception { patternTree.constructTree(); ; ClusterSchemaTree schemas = - schemaRegion.fetchSchema(patternTree, Collections.EMPTY_MAP, true, true); + schemaRegion.fetchSeriesSchema( + patternTree, Collections.EMPTY_MAP, true, false, true, false); List measurementPaths = schemas.searchMeasurementPaths(new PartialPath("root.sg.wf01.wt01.*")).left; Assert.assertEquals(measurementPaths.size(), 2); @@ -182,7 +183,9 @@ public void testFetchSchema() throws Exception { patternTree = new PathPatternTree(); patternTree.appendPathPattern(new PartialPath("root.sg.wf01.wt01.temp")); patternTree.constructTree(); - schemas = schemaRegion.fetchSchema(patternTree, Collections.EMPTY_MAP, false, true); + schemas = + schemaRegion.fetchSeriesSchema( + patternTree, Collections.EMPTY_MAP, false, false, true, false); measurementPaths = schemas.searchMeasurementPaths(new PartialPath("root.sg.wf01.wt01.temp")).left; Assert.assertEquals(measurementPaths.size(), 1); @@ -368,7 +371,7 @@ public void testDeleteTimeseries() throws Exception { schemaRegion.deleteTimeseriesInBlackList(patternTree); List schemas = schemaRegion - .fetchSchema(ALL_MATCH_SCOPE, Collections.EMPTY_MAP, false, true) + .fetchSeriesSchema(ALL_MATCH_SCOPE, Collections.EMPTY_MAP, false, false, true, false) .searchMeasurementPaths(ALL_MATCH_PATTERN) .left; Assert.assertEquals(1, schemas.size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java index d85d203ec64d..0811e3c35efd 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionSimpleRecoverTest.java @@ -84,15 +84,25 @@ public void testRecoverWithAlignedTemplate() throws Exception { new PartialPath("root.sg.d1"), 2, templateId), template); ClusterSchemaTree schemaTree = - schemaRegion.fetchSchema( - ALL_MATCH_SCOPE, Collections.singletonMap(templateId, template), true, true); + schemaRegion.fetchSeriesSchema( + ALL_MATCH_SCOPE, + Collections.singletonMap(templateId, template), + true, + false, + true, + false); Assert.assertTrue(schemaTree.getAllDevices().get(0).isAligned()); simulateRestart(); schemaRegion = getSchemaRegion("root.sg", 0); schemaTree = - schemaRegion.fetchSchema( - ALL_MATCH_SCOPE, Collections.singletonMap(templateId, template), true, true); + schemaRegion.fetchSeriesSchema( + ALL_MATCH_SCOPE, + Collections.singletonMap(templateId, template), + true, + false, + true, + false); Assert.assertTrue(schemaTree.getAllDevices().get(0).isAligned()); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java index 3aa6d9019fdb..555a081f0015 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionTemplateTest.java @@ -77,8 +77,13 @@ public void testFetchNestedTemplateDevice() throws Exception { new PartialPath("root.sg.d1.GPS"), 3, templateId), template); ClusterSchemaTree schemaTree = - schemaRegion.fetchSchema( - ALL_MATCH_SCOPE, Collections.singletonMap(templateId, template), true, true); + schemaRegion.fetchSeriesSchema( + ALL_MATCH_SCOPE, + Collections.singletonMap(templateId, template), + true, + false, + true, + false); Assert.assertEquals(2, schemaTree.getAllDevices().size()); for (DeviceSchemaInfo deviceSchemaInfo : schemaTree.getAllDevices()) { Assert.assertEquals(templateId, deviceSchemaInfo.getTemplateId()); @@ -213,7 +218,7 @@ public void testFetchSchemaWithTemplate() throws Exception { // check fetch schema ClusterSchemaTree schemaTree = - schemaRegion.fetchSchema(ALL_MATCH_SCOPE, templateMap, true, true); + schemaRegion.fetchSeriesSchema(ALL_MATCH_SCOPE, templateMap, true, false, true, false); schemaTree.setTemplateMap(templateMap); List schemas = schemaTree.searchMeasurementPaths(ALL_MATCH_PATTERN).left; Assert.assertEquals(expectedTimeseries.size(), schemas.size()); @@ -258,16 +263,16 @@ public void testDeleteSchemaWithTemplate() throws Exception { patternTree.appendFullPath(new PartialPath("root.db.d1.s1")); ClusterSchemaTree schemaTree = - schemaRegion.fetchSchema( - patternTree, Collections.singletonMap(templateId, template), false, true); + schemaRegion.fetchSeriesSchema( + patternTree, Collections.singletonMap(templateId, template), false, false, true, false); schemaTree.setTemplateMap(Collections.singletonMap(templateId, template)); Assert.assertEquals( 1, schemaTree.searchMeasurementPaths(new PartialPath("root.db.d1.s1")).left.size()); patternTree = new PathPatternTree(); patternTree.appendFullPath(new PartialPath("root.db.d1.s3")); schemaTree = - schemaRegion.fetchSchema( - patternTree, Collections.singletonMap(templateId, template), false, true); + schemaRegion.fetchSeriesSchema( + patternTree, Collections.singletonMap(templateId, template), false, false, true, false); schemaTree.setTemplateMap(Collections.singletonMap(templateId, template)); Assert.assertEquals( 0, schemaTree.searchMeasurementPaths(new PartialPath("root.db.d1.s3")).left.size()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java index e221fd7a6d11..b1b0cae80697 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/common/schematree/ClusterSchemaTreeTest.java @@ -1030,6 +1030,7 @@ public void testHasView() throws IllegalPathException { new MeasurementSchema("s1", TSDataType.INT32), null, null, + null, false); Assert.assertFalse(schemaTree.hasLogicalViewMeasurement()); schemaTree.appendSingleMeasurement( @@ -1037,6 +1038,7 @@ public void testHasView() throws IllegalPathException { new LogicalViewSchema("s1", new TimeSeriesViewOperand("root.db.d.s1")), null, null, + null, false); Assert.assertTrue(schemaTree.hasLogicalViewMeasurement()); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java index 52c5a71a46f0..335dd31c74aa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/OperatorMemoryTest.java @@ -832,13 +832,15 @@ public void SchemaFetchScanOperatorTest() { driverContext.addOperatorContext(1, planNodeId, SeriesScanOperator.class.getSimpleName()); SchemaFetchScanOperator operator = - new SchemaFetchScanOperator( + SchemaFetchScanOperator.ofSeries( planNodeId, driverContext.getOperatorContexts().get(0), null, null, null, false, + false, + false, false); assertEquals(DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES, operator.calculateMaxPeekMemory()); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java index 8286a11c51a2..50249bc55d52 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/SchemaFetchScanOperatorTest.java @@ -59,8 +59,16 @@ public void testSchemaFetchResult() throws Exception { patternTree.constructTree(); SchemaFetchScanOperator schemaFetchScanOperator = - new SchemaFetchScanOperator( - null, null, patternTree, Collections.emptyMap(), schemaRegion, false, true); + SchemaFetchScanOperator.ofSeries( + null, + null, + patternTree, + Collections.emptyMap(), + schemaRegion, + false, + false, + true, + false); Assert.assertTrue(schemaFetchScanOperator.hasNext()); @@ -151,7 +159,9 @@ private ISchemaRegion mockSchemaRegion() throws Exception { patternTree.appendPathPattern(new PartialPath("root.**.status")); patternTree.appendPathPattern(new PartialPath("root.**.s1")); patternTree.constructTree(); - Mockito.when(schemaRegion.fetchSchema(patternTree, Collections.emptyMap(), false, true)) + Mockito.when( + schemaRegion.fetchSeriesSchema( + patternTree, Collections.emptyMap(), false, false, true, false)) .thenReturn(clusterSchemaTree); return schemaRegion; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java index 718d10ef8156..1e6042e05dfe 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/analyze/FakeSchemaFetcherImpl.java @@ -53,6 +53,20 @@ public ClusterSchemaTree fetchSchema( return schemaTree; } + @Override + public ISchemaTree fetchRawSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + schemaTree.setDatabases(Collections.singleton("root.sg")); + return schemaTree; + } + + @Override + public ISchemaTree fetchRawSchemaInMeasurementLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + schemaTree.setDatabases(Collections.singleton("root.sg")); + return schemaTree; + } + @Override public ISchemaTree fetchSchemaWithTags( PathPatternTree patternTree, boolean withTemplate, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java index 59cd36a2f8ae..e08bbb074ea2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util.java @@ -305,6 +305,18 @@ public ISchemaTree fetchSchema( return ANALYSIS.getSchemaTree(); } + @Override + public ISchemaTree fetchRawSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return ANALYSIS.getSchemaTree(); + } + + @Override + public ISchemaTree fetchRawSchemaInMeasurementLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return ANALYSIS.getSchemaTree(); + } + @Override public ISchemaTree fetchSchemaWithTags( PathPatternTree patternTree, boolean withTemplate, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java index eb802edd9e54..3d5e66aea49b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/Util2.java @@ -199,6 +199,18 @@ public ISchemaTree fetchSchema( return ANALYSIS.getSchemaTree(); } + @Override + public ISchemaTree fetchRawSchemaInDeviceLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return ANALYSIS.getSchemaTree(); + } + + @Override + public ISchemaTree fetchRawSchemaInMeasurementLevel( + PathPatternTree patternTree, PathPatternTree authorityScope, MPPQueryContext context) { + return ANALYSIS.getSchemaTree(); + } + @Override public ISchemaTree fetchSchemaWithTags( PathPatternTree patternTree, boolean withTemplate, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java index 452e6dce7983..4bee7e70c27c 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/logical/RegionScanLogicalPlannerTest.java @@ -23,8 +23,10 @@ import org.apache.iotdb.commons.path.AlignedPath; import org.apache.iotdb.commons.path.MeasurementPath; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.db.queryengine.common.DeviceContext; import org.apache.iotdb.db.queryengine.common.QueryId; -import org.apache.iotdb.db.queryengine.common.TimeseriesSchemaInfo; +import org.apache.iotdb.db.queryengine.common.TimeseriesContext; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -49,10 +51,10 @@ public class RegionScanLogicalPlannerTest { - private static Map>> + private static Map>> deviceToTimeseriesSchemaInfoMap; - private static Map>> + private static Map>> getDeviceToTimeseriesSchemaInfoMap() throws IllegalPathException { if (deviceToTimeseriesSchemaInfoMap != null) { @@ -60,58 +62,57 @@ public class RegionScanLogicalPlannerTest { } deviceToTimeseriesSchemaInfoMap = new HashMap<>(); - Map> timeseriesSchemaInfoMap = new HashMap<>(); + Map> timeseriesSchemaInfoMap = new HashMap<>(); timeseriesSchemaInfoMap.put( new MeasurementPath("root.sg.d1.s1", TSDataType.INT32), Collections.singletonList( - new TimeseriesSchemaInfo( - "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); + new TimeseriesContext( + "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null))); timeseriesSchemaInfoMap.put( new MeasurementPath("root.sg.d1.s2", TSDataType.DOUBLE), Collections.singletonList( - new TimeseriesSchemaInfo( - "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); + new TimeseriesContext( + "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null))); timeseriesSchemaInfoMap.put( new MeasurementPath("root.sg.d1.s3", TSDataType.BOOLEAN), Collections.singletonList( - new TimeseriesSchemaInfo( - "BOOLEAN", null, "PLAIN", "LZ4", "{\"key1\":\"value2\"}", null, null))); + new TimeseriesContext( + "BOOLEAN", null, "PLAIN", "LZ4", "{\"key1\":\"value2\"}", null, null, null))); deviceToTimeseriesSchemaInfoMap.put( new PartialPath(new PlainDeviceID("root.sg.d1")), timeseriesSchemaInfoMap); - Map> timeseriesSchemaInfoMap2 = new HashMap<>(); + Map> timeseriesSchemaInfoMap2 = new HashMap<>(); timeseriesSchemaInfoMap2.put( new MeasurementPath("root.sg.d2.s1", TSDataType.INT32), Collections.singletonList( - new TimeseriesSchemaInfo( - "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); + new TimeseriesContext( + "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null))); timeseriesSchemaInfoMap2.put( new MeasurementPath("root.sg.d2.s2", TSDataType.DOUBLE), Collections.singletonList( - new TimeseriesSchemaInfo( - "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null))); + new TimeseriesContext( + "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null))); timeseriesSchemaInfoMap2.put( new MeasurementPath("root.sg.d2.s4", TSDataType.TEXT), Collections.singletonList( - new TimeseriesSchemaInfo( - "TEXT", null, "PLAIN", "LZ4", "{\"key2\":\"value1\"}", null, null))); + new TimeseriesContext( + "TEXT", null, "PLAIN", "LZ4", "{\"key2\":\"value1\"}", null, null, null))); deviceToTimeseriesSchemaInfoMap.put( new PartialPath(new PlainDeviceID("root.sg.d2")), timeseriesSchemaInfoMap2); List schemas = new ArrayList<>(); schemas.add("s1"); schemas.add("s2"); - List timeseriesSchemaInfoList = new ArrayList<>(); - Map> timeseriesSchemaInfoMap3 = new HashMap<>(); - timeseriesSchemaInfoList.add( - new TimeseriesSchemaInfo( - "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null)); - timeseriesSchemaInfoList.add( - new TimeseriesSchemaInfo( - "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null)); + List timeseriesContextList = new ArrayList<>(); + Map> timeseriesSchemaInfoMap3 = new HashMap<>(); + timeseriesContextList.add( + new TimeseriesContext( + "INT32", null, "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null)); + timeseriesContextList.add( + new TimeseriesContext( + "DOUBLE", "status", "PLAIN", "LZ4", "{\"key1\":\"value1\"}", null, null, null)); timeseriesSchemaInfoMap3.put( - new AlignedPath("root.sg.d2.a", schemas, Collections.emptyList()), - timeseriesSchemaInfoList); + new AlignedPath("root.sg.d2.a", schemas, Collections.emptyList()), timeseriesContextList); deviceToTimeseriesSchemaInfoMap.put( new PartialPath(new PlainDeviceID("root.sg.d2.a")), timeseriesSchemaInfoMap3); @@ -126,13 +127,19 @@ public void testShowDevicesWithTimeCondition() throws IllegalPathException { // fake initResultNodeContext() queryId.genPlanNodeId(); - Map devicePathsToAligned = new HashMap<>(); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d1")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2.a")), true); + Map deviceContextMap = new HashMap<>(); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d1")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2.a")), + new DeviceContext(true, SchemaConstant.NON_TEMPLATE)); DeviceRegionScanNode regionScanNode = - new DeviceRegionScanNode(queryId.genPlanNodeId(), devicePathsToAligned, false, null); + new DeviceRegionScanNode(queryId.genPlanNodeId(), deviceContextMap, false, null); PlanNode actualPlan = parseSQLToPlanNode(sql); Assert.assertEquals(actualPlan, regionScanNode); @@ -146,13 +153,19 @@ public void testShowDevicesWithTimeConditionWithLimitOffset() throws IllegalPath // fake initResultNodeContext() queryId.genPlanNodeId(); - Map devicePathsToAligned = new HashMap<>(); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d1")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2.a")), true); + Map deviceContextMap = new HashMap<>(); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d1")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2.a")), + new DeviceContext(true, SchemaConstant.NON_TEMPLATE)); DeviceRegionScanNode regionScanNode = - new DeviceRegionScanNode(queryId.genPlanNodeId(), devicePathsToAligned, false, null); + new DeviceRegionScanNode(queryId.genPlanNodeId(), deviceContextMap, false, null); LimitNode limitNode = new LimitNode(queryId.genPlanNodeId(), 20); limitNode.addChild(regionScanNode); @@ -171,13 +184,19 @@ public void testCountDevicesWithTimeConditionWithLimitOffset() throws IllegalPat // fake initResultNodeContext() queryId.genPlanNodeId(); - Map devicePathsToAligned = new HashMap<>(); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d1")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2")), false); - devicePathsToAligned.put(new PartialPath(new PlainDeviceID("root.sg.d2.a")), true); + Map deviceContextMap = new HashMap<>(); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d1")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2")), + new DeviceContext(false, SchemaConstant.NON_TEMPLATE)); + deviceContextMap.put( + new PartialPath(new PlainDeviceID("root.sg.d2.a")), + new DeviceContext(true, SchemaConstant.NON_TEMPLATE)); DeviceRegionScanNode regionScanNode = - new DeviceRegionScanNode(queryId.genPlanNodeId(), devicePathsToAligned, true, null); + new DeviceRegionScanNode(queryId.genPlanNodeId(), deviceContextMap, true, null); PlanNode actualPlan = parseSQLToPlanNode(sql); Assert.assertEquals(actualPlan, regionScanNode); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/DeviceSchemaFetchScanNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/DeviceSchemaFetchScanNodeTest.java new file mode 100644 index 000000000000..ca52035bb504 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/DeviceSchemaFetchScanNodeTest.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.planner.node.metadata.read; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.DeviceSchemaFetchScanNode; + +import org.junit.Assert; +import org.junit.Test; + +import java.nio.ByteBuffer; + +public class DeviceSchemaFetchScanNodeTest { + + @Test + public void testSerialization() throws IllegalPathException { + PathPatternTree patternTree = new PathPatternTree(); + patternTree.appendPathPattern(new PartialPath("root.sg.**.*")); + DeviceSchemaFetchScanNode deviceSchemaFetchScanNodeTest = + new DeviceSchemaFetchScanNode( + new PlanNodeId("0"), new PartialPath("root.sg"), patternTree, patternTree); + ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 1024); + deviceSchemaFetchScanNodeTest.serialize(byteBuffer); + byteBuffer.flip(); + DeviceSchemaFetchScanNode recoveredNode = + (DeviceSchemaFetchScanNode) PlanNodeType.deserialize(byteBuffer); + Assert.assertEquals("root.sg", recoveredNode.getStorageGroup().getFullPath()); + Assert.assertEquals( + "root.sg.**.*", recoveredNode.getPatternTree().getAllPathPatterns().get(0).getFullPath()); + Assert.assertEquals( + "root.sg.**.*", + recoveredNode.getAuthorityScope().getAllPathPatterns().get(0).getFullPath()); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java index c242778efab6..e31d886a4f92 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchMergeNodeTest.java @@ -25,7 +25,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchMergeNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.junit.Assert; import org.junit.Test; @@ -42,15 +42,17 @@ public void testSerialization() throws IllegalPathException { new SchemaFetchMergeNode(new PlanNodeId("0"), Arrays.asList("root.db1", "root.db2")); PathPatternTree patternTree = new PathPatternTree(); patternTree.appendPathPattern(new PartialPath("root.sg.**.*")); - SchemaFetchScanNode schemaFetchScanNode = - new SchemaFetchScanNode( + SeriesSchemaFetchScanNode seriesSchemaFetchScanNode = + new SeriesSchemaFetchScanNode( new PlanNodeId("0"), new PartialPath("root.sg"), patternTree, Collections.emptyMap(), true, - true); - schemaFetchMergeNode.addChild(schemaFetchScanNode); + false, + true, + false); + schemaFetchMergeNode.addChild(seriesSchemaFetchScanNode); ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 1024); schemaFetchMergeNode.serialize(byteBuffer); byteBuffer.flip(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchScanNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SeriesSchemaFetchScanNodeTest.java similarity index 83% rename from iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchScanNodeTest.java rename to iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SeriesSchemaFetchScanNodeTest.java index 0d45a127f176..0ad52409c09d 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SchemaFetchScanNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/planner/node/metadata/read/SeriesSchemaFetchScanNodeTest.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SchemaFetchScanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.metedata.read.SeriesSchemaFetchScanNode; import org.junit.Assert; import org.junit.Test; @@ -32,24 +32,27 @@ import java.nio.ByteBuffer; import java.util.Collections; -public class SchemaFetchScanNodeTest { +public class SeriesSchemaFetchScanNodeTest { @Test public void testSerialization() throws IllegalPathException { PathPatternTree patternTree = new PathPatternTree(); patternTree.appendPathPattern(new PartialPath("root.sg.**.*")); - SchemaFetchScanNode schemaFetchScanNode = - new SchemaFetchScanNode( + SeriesSchemaFetchScanNode seriesSchemaFetchScanNode = + new SeriesSchemaFetchScanNode( new PlanNodeId("0"), new PartialPath("root.sg"), patternTree, Collections.emptyMap(), true, - true); + false, + true, + false); ByteBuffer byteBuffer = ByteBuffer.allocate(1024 * 1024); - schemaFetchScanNode.serialize(byteBuffer); + seriesSchemaFetchScanNode.serialize(byteBuffer); byteBuffer.flip(); - SchemaFetchScanNode recoveredNode = (SchemaFetchScanNode) PlanNodeType.deserialize(byteBuffer); + SeriesSchemaFetchScanNode recoveredNode = + (SeriesSchemaFetchScanNode) PlanNodeType.deserialize(byteBuffer); Assert.assertEquals("root.sg", recoveredNode.getStorageGroup().getFullPath()); Assert.assertEquals( "root.sg.**.*", recoveredNode.getPatternTree().getAllPathPatterns().get(0).getFullPath()); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTreeUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTreeUtils.java index 2fa1b16f2663..ac124fe67ccf 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTreeUtils.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PathPatternTreeUtils.java @@ -22,6 +22,11 @@ import org.apache.iotdb.commons.schema.SchemaConstant; public class PathPatternTreeUtils { + + private PathPatternTreeUtils() { + // hidden constructor + } + /** * Intersect the pattern tree with the full path prefix tree, and return the intersected pattern * tree.