diff --git a/distribution/src/assembly/all.xml b/distribution/src/assembly/all.xml index 028af358eac4..9a236d5725a1 100644 --- a/distribution/src/assembly/all.xml +++ b/distribution/src/assembly/all.xml @@ -57,6 +57,15 @@ conf ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf + + **/*.sh + **/*.bat + **/iotdb-system.properties + + + + conf + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/target/conf sbin diff --git a/distribution/src/assembly/confignode.xml b/distribution/src/assembly/confignode.xml index 502b90758cae..3944c62325ef 100644 --- a/distribution/src/assembly/confignode.xml +++ b/distribution/src/assembly/confignode.xml @@ -54,6 +54,15 @@ conf ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf + + **/*.sh + **/*.bat + **/iotdb-system.properties + + + + conf + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/target/conf diff --git a/distribution/src/assembly/datanode.xml b/distribution/src/assembly/datanode.xml index 84697fda6d74..9b0ae34a2bbb 100644 --- a/distribution/src/assembly/datanode.xml +++ b/distribution/src/assembly/datanode.xml @@ -45,6 +45,15 @@ conf ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/src/assembly/resources/conf + + **/*.sh + **/*.bat + **/iotdb-system.properties + + + + conf + ${maven.multiModuleProjectDirectory}/iotdb-core/node-commons/target/conf sbin diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java index 25d2935b9447..15ad2983ca87 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRepairDataIT.java @@ -60,7 +60,7 @@ public static void setUp() throws Exception { @AfterClass public static void tearDown() throws Exception { - EnvFactory.getEnv().initClusterEnvironment(); + EnvFactory.getEnv().cleanClusterEnvironment(); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java new file mode 100644 index 000000000000..3ab24dd94f70 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSetConfigurationIT.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it; + +import org.apache.iotdb.commons.conf.CommonConfig; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.Statement; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class}) +public class IoTDBSetConfigurationIT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testSetConfiguration() throws Exception { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("set configuration \"enable_seq_space_compaction\"=\"false\""); + statement.execute("set configuration \"enable_unseq_space_compaction\"=\"false\" on 0"); + statement.execute("set configuration \"enable_cross_space_compaction\"=\"false\" on 1"); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + for (ConfigNodeWrapper configNodeWrapper : EnvFactory.getEnv().getConfigNodeWrapperList()) { + String systemPropertiesPath = + configNodeWrapper.getNodePath() + + File.separator + + "conf" + + File.separator + + CommonConfig.SYSTEM_CONFIG_NAME; + File f = new File(systemPropertiesPath); + String content = new String(Files.readAllBytes(f.toPath())); + Assert.assertTrue(content.contains("enable_seq_space_compaction=false")); + Assert.assertTrue(content.contains("enable_unseq_space_compaction=false")); + } + for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) { + String systemPropertiesPath = + dataNodeWrapper.getNodePath() + + File.separator + + "conf" + + File.separator + + CommonConfig.SYSTEM_CONFIG_NAME; + File f = new File(systemPropertiesPath); + String content = new String(Files.readAllBytes(f.toPath())); + Assert.assertTrue(content.contains("enable_seq_space_compaction=false")); + Assert.assertTrue(content.contains("enable_cross_space_compaction=false")); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java index f173e2ff9d60..d800bb8a054a 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSettleIT.java @@ -56,7 +56,7 @@ public static void setUp() throws Exception { @AfterClass public static void tearDown() throws Exception { close(); - EnvFactory.getEnv().initClusterEnvironment(); + EnvFactory.getEnv().cleanClusterEnvironment(); } @Test diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java index fe8c9c6c39e4..19b487193834 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBSimpleQueryIT.java @@ -39,7 +39,6 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; -import java.sql.Timestamp; import java.sql.Types; import java.time.LocalDate; import java.util.ArrayList; @@ -1173,13 +1172,13 @@ public void testNewDataType() { while (resultSet.next()) { long time = resultSet.getLong(1); Date date = resultSet.getDate(2); - Timestamp timestamp = resultSet.getTimestamp(3); + long timestamp = resultSet.getLong(3); byte[] blob = resultSet.getBytes(4); String text = resultSet.getString(5); assertEquals(2024 - 1900, date.getYear()); assertEquals(5 - 1, date.getMonth()); assertEquals(time % 31 + 1, date.getDate()); - assertEquals(time, timestamp.getTime()); + assertEquals(time, timestamp); assertArrayEquals(byteArray, blob); assertEquals(String.valueOf(time), text); } diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java index bb2b1d539bff..8eb1026121aa 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/IoTDBAggregationIT.java @@ -348,8 +348,8 @@ public void countTest() { public void firstTest() { String[] retArray = new String[] { - "0,2000,2000,2000.0,2000,2000,0x2000,2000-01-01,1970-01-01T00:00:02.000Z", - "0,500,500,500.0,500,500,0x0500,1500-01-01,1970-01-01T00:00:00.500Z" + "0,2000,2000,2000.0,2000,2000,0x2000,2000-01-01,2000", + "0,500,500,500.0,500,500,0x0500,1500-01-01,500" }; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -453,9 +453,9 @@ public void firstTest() { public void lastTest() { String[] retArray = new String[] { - "0,8499,8499.0,8499,0x8499,8499-01-01,1970-01-01T00:00:08.499Z", - "0,1499,1499.0,1499,0x1499,1499-01-01,1970-01-01T00:00:01.499Z", - "0,2200,2200.0,2200,0x2200,2200-01-01,1970-01-01T00:00:02.200Z" + "0,8499,8499.0,8499,0x8499,8499-01-01,8499", + "0,1499,1499.0,1499,0x1499,1499-01-01,1499", + "0,2200,2200.0,2200,0x2200,2200-01-01,2200" }; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { @@ -727,10 +727,7 @@ public void firstLastValueTest() throws SQLException { @Test public void maxminValueTest() { String[] retArray = - new String[] { - "0,8499,500.0,999,1000,8499-01-01,1000-01-01,1970-01-01T00:00:08.499Z,1970-01-01T00:00:00.500Z", - "0,2499,500.0" - }; + new String[] {"0,8499,500.0,999,1000,8499-01-01,1000-01-01,8499,500", "0,2499,500.0"}; try (Connection connection = EnvFactory.getEnv().getConnection(); Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java index 1ccdc8c1195e..ccf435f83620 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/maxby/IoTDBMaxByIT.java @@ -210,8 +210,7 @@ public void testMaxByWithDifferentXAndYInputTypes() { "root.db.d1", new String[] {"x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10"}, new String[] {"y1", "y2", "y3", "y4", "y7", "y9", "y10"}); - String[] retArray = - new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,1970-01-01T00:00:00.003Z,"}; + String[] retArray = new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,3,"}; for (Map.Entry expectedHeader : expectedHeaders.entrySet()) { String y = expectedHeader.getKey(); resultSetEqualTest( diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java index 82867e782ccd..77cb86e62d4d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/aggregation/minby/IoTDBMinByIT.java @@ -210,8 +210,7 @@ public void testMaxByWithDifferentXAndYInputTypes() { "root.db.d1", new String[] {"x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10"}, new String[] {"y1", "y2", "y3", "y4", "y7", "y9", "y10"}); - String[] retArray = - new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,1970-01-01T00:00:00.003Z,"}; + String[] retArray = new String[] {"3,3,3.0,3.0,false,3,3,0x33,2024-01-03,3,"}; for (Map.Entry expectedHeader : expectedHeaders.entrySet()) { String y = expectedHeader.getKey(); resultSetEqualTest( diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFWindowQuery2IT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFWindowQuery2IT.java new file mode 100644 index 000000000000..c2a2db78b36d --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDFWindowQuery2IT.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it.udf; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.constant.UDFTestConstant; + +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({ClusterIT.class}) +public class IoTDBUDFWindowQuery2IT { + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv() + .getConfig() + .getCommonConfig() + .setEnableSeqSpaceCompaction(false) + .setEnableUnseqSpaceCompaction(false) + .setEnableCrossSpaceCompaction(false) + .setUdfMemoryBudgetInMB(5); + EnvFactory.getEnv().initClusterEnvironment(); + createTimeSeries(); + generateData(); + registerUDF(); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + private static void createTimeSeries() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("CREATE DATABASE root.sg"); + statement.execute("CREATE TIMESERIES root.sg.d1.s1 with datatype=INT32,encoding=PLAIN"); + statement.execute("CREATE TIMESERIES root.sg.d1.s2 with datatype=INT32,encoding=PLAIN"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + private static void generateData() { + // SessionWindow + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute("INSERT INTO root.sg.d1(time, s1, s2) VALUES (1, 1, 1)"); + statement.execute("INSERT INTO root.sg.d1(time, s1, s2) VALUES (2, 2, 2)"); + statement.execute("INSERT INTO root.sg.d1(time, s1, s2) VALUES (3, 3, 3)"); + statement.execute("INSERT INTO root.sg.d1(time, s1, s2) VALUES (9, 9, 9)"); + statement.execute("INSERT INTO root.sg.d1(time, s1, s2) VALUES (5, 5, 5)"); + statement.execute("INSERT INTO root.sg.d1(time, s1, s2) VALUES (12, 12, 12)"); + statement.execute("INSERT INTO root.sg.d1(time, s1, s2) VALUES (14, 14, 14)"); + statement.execute("INSERT INTO root.sg.d1(time, s1, s2) VALUES (18, 18, 18)"); + statement.execute("INSERT INTO root.sg.d1(time, s1, s2) VALUES (21, 21, 21)"); + statement.execute("INSERT INTO root.sg.d1(time, s1, s2) VALUES (24, 24, 24)"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + private static void registerUDF() { + try (Connection connection = EnvFactory.getEnv().getConnection(); + Statement statement = connection.createStatement()) { + statement.execute( + "CREATE FUNCTION window_start_end AS 'org.apache.iotdb.db.query.udf.example.WindowStartEnd'"); + } catch (SQLException throwable) { + fail(throwable.getMessage()); + } + } + + @Test + public void testSessionTimeWindow1() { + long[] windowStart = new long[] {12, 18}; + long[] windowEnd = new long[] {14, 21}; + testSessionTimeWindowSS("3", windowStart, windowEnd, 12L, 24L); + } + + @Test + public void testSessionTimeWindow2() { + long[] windowStart = new long[] {12, 18}; + long[] windowEnd = new long[] {14, 24}; + testSessionTimeWindowSS("3", windowStart, windowEnd, 12L, Long.MAX_VALUE); + } + + @Test + public void testStateTimeWindow() { + long[] windowStart = new long[] {12, 18}; + long[] windowEnd = new long[] {14, 21}; + testStateWindowSS("3", windowStart, windowEnd, 12L, 24L); + } + + private void testSessionTimeWindowSS( + String sessionGap, long[] windowStart, long[] windowEnd, Long displayBegin, Long displayEnd) { + String sql; + if (displayBegin == null) { + sql = + String.format( + "select window_start_end(s1, '%s'='%s', '%s'='%s') from root.sg.d1", + UDFTestConstant.ACCESS_STRATEGY_KEY, + UDFTestConstant.ACCESS_STRATEGY_SESSION, + UDFTestConstant.SESSION_GAP_KEY, + sessionGap); + } else { + sql = + String.format( + "select window_start_end(s1, '%s'='%s', '%s'='%s', '%s'='%s', '%s'='%s') from root.sg.d1", + UDFTestConstant.ACCESS_STRATEGY_KEY, + UDFTestConstant.ACCESS_STRATEGY_SESSION, + UDFTestConstant.DISPLAY_WINDOW_BEGIN_KEY, + displayBegin, + UDFTestConstant.DISPLAY_WINDOW_END_KEY, + displayEnd, + UDFTestConstant.SESSION_GAP_KEY, + sessionGap); + } + + try (Connection conn = EnvFactory.getEnv().getConnection(); + Statement statement = conn.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + assertEquals(2, resultSet.getMetaData().getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + Assert.assertEquals(resultSet.getLong(1), windowStart[cnt]); + Assert.assertEquals(resultSet.getLong(2), windowEnd[cnt]); + cnt++; + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private void testStateWindowSS( + String delta, long[] windowStart, long[] windowEnd, Long displayBegin, Long displayEnd) { + String sql; + if (displayBegin == null) { + if (delta == null) { + sql = + String.format( + "select window_start_end(%s, '%s'='%s') from root.sg.d1", + "s2", UDFTestConstant.ACCESS_STRATEGY_KEY, UDFTestConstant.ACCESS_STRATEGY_STATE); + } else { + sql = + String.format( + "select window_start_end(%s, '%s'='%s', '%s'='%s') from root.sg.d1", + "s2", + UDFTestConstant.ACCESS_STRATEGY_KEY, + UDFTestConstant.ACCESS_STRATEGY_STATE, + UDFTestConstant.STATE_DELTA_KEY, + delta); + } + } else { + sql = + String.format( + "select window_start_end(%s, '%s'='%s', '%s'='%s', '%s'='%s', '%s'='%s') from root.sg.d1", + "s2", + UDFTestConstant.ACCESS_STRATEGY_KEY, + UDFTestConstant.ACCESS_STRATEGY_STATE, + UDFTestConstant.DISPLAY_WINDOW_BEGIN_KEY, + displayBegin, + UDFTestConstant.DISPLAY_WINDOW_END_KEY, + displayEnd, + UDFTestConstant.STATE_DELTA_KEY, + delta); + } + + try (Connection conn = EnvFactory.getEnv().getConnection(); + Statement statement = conn.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + assertEquals(2, resultSet.getMetaData().getColumnCount()); + int cnt = 0; + while (resultSet.next()) { + Assert.assertEquals(resultSet.getLong(1), windowStart[cnt]); + Assert.assertEquals(resultSet.getLong(2), windowEnd[cnt]); + cnt++; + } + } catch (SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java index 61de72829ccc..2ddc99f6ead5 100644 --- a/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/udf/IoTDBUDTFBuiltinFunctionIT.java @@ -205,7 +205,7 @@ public void testSelectorFunctions() { } else if (j == 5) { assertEquals("2024-01-0" + (i + 1), resultSet.getString(7)); } else if (j == 6) { - assertEquals(String.format("1970-01-01T00:00:00.00%dZ", i), resultSet.getString(8)); + assertEquals(String.valueOf(i), resultSet.getString(8)); } } } @@ -251,7 +251,7 @@ public void testSelectorFunctions() { } else if (j == 5) { assertEquals("2024-01-0" + (i + 1), resultSet.getString(7)); } else if (j == 6) { - assertEquals(String.format("1970-01-01T00:00:00.00%dZ", i), resultSet.getString(8)); + assertEquals(String.valueOf(i), resultSet.getString(8)); } } } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java index bb1d2a2e25ea..88425f7e474a 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/cli/AbstractCli.java @@ -32,6 +32,9 @@ import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.lang3.ArrayUtils; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.BytesUtils; +import org.apache.tsfile.utils.DateUtils; import java.io.BufferedReader; import java.io.IOException; @@ -679,7 +682,7 @@ private static List> cacheResult( RpcUtils.formatDatetime( timeFormat, timestampPrecision, resultSet.getLong(i), zoneId); } else { - tmp = resultSet.getString(i); + tmp = getStringByColumnIndex(ioTDBJDBCResultSet, i, zoneId); } if (tmp == null) { tmp = NULL; @@ -729,6 +732,35 @@ private static List> cacheResult( return lists; } + private static String getStringByColumnIndex( + IoTDBJDBCResultSet resultSet, int columnIndex, ZoneId zoneId) throws SQLException { + TSDataType type = TSDataType.valueOf(resultSet.getColumnTypeByIndex(columnIndex)); + switch (type) { + case BOOLEAN: + return String.valueOf(resultSet.getBoolean(columnIndex)); + case INT32: + return String.valueOf(resultSet.getInt(columnIndex)); + case INT64: + return String.valueOf(resultSet.getLong(columnIndex)); + case FLOAT: + return String.valueOf(resultSet.getFloat(columnIndex)); + case DOUBLE: + return String.valueOf(resultSet.getDouble(columnIndex)); + case TEXT: + case STRING: + return resultSet.getString(columnIndex); + case BLOB: + return BytesUtils.parseBlobByteArrayToString(resultSet.getBytes(columnIndex)); + case DATE: + return DateUtils.formatDate(resultSet.getInt(columnIndex)); + case TIMESTAMP: + return RpcUtils.formatDatetime( + timeFormat, timestampPrecision, resultSet.getLong(columnIndex), zoneId); + default: + return null; + } + } + private static List> cacheTracingInfo(ResultSet resultSet, List maxSizeList) throws Exception { List> lists = new ArrayList<>(2); diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java index ab6009343aa5..269d6ec7c267 100644 --- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java +++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java @@ -1338,4 +1338,12 @@ public List getColumns() { public List getSgColumns() { return sgColumns; } + + public String getColumnTypeByIndex(int columnIndex) { + if (!isIgnoreTimeStamp() && columnIndex == 1) { + return TSDataType.TIMESTAMP.name(); + } + + return ioTDBRpcDataSet.columnTypeList.get(columnIndex - 1); + } } diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java index cca84cc8c330..5bba6d0ea718 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java @@ -587,6 +587,7 @@ public String getString(int index, TSDataType tsDataType, byte[][] values) { case INT32: return String.valueOf(BytesUtils.bytesToInt(values[index])); case INT64: + case TIMESTAMP: return String.valueOf(BytesUtils.bytesToLong(values[index])); case FLOAT: return String.valueOf(BytesUtils.bytesToFloat(values[index])); @@ -597,9 +598,6 @@ public String getString(int index, TSDataType tsDataType, byte[][] values) { return new String(values[index], StandardCharsets.UTF_8); case BLOB: return BytesUtils.parseBlobByteArrayToString(values[index]); - case TIMESTAMP: - return RpcUtils.formatDatetime( - RpcUtils.DEFAULT_TIME_FORMAT, "ms", BytesUtils.bytesToLong(values[index]), zoneId); case DATE: return DateUtils.formatDate(BytesUtils.bytesToInt(values[index])); default: diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java index 014c5eedae1f..9eb33fc36b2e 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java @@ -562,6 +562,7 @@ public String getString(int index, TSDataType tsDataType) { case INT32: return String.valueOf(curTsBlock.getColumn(index).getInt(tsBlockIndex)); case INT64: + case TIMESTAMP: return String.valueOf(curTsBlock.getColumn(index).getLong(tsBlockIndex)); case FLOAT: return String.valueOf(curTsBlock.getColumn(index).getFloat(tsBlockIndex)); @@ -576,9 +577,6 @@ public String getString(int index, TSDataType tsDataType) { case BLOB: return BytesUtils.parseBlobByteArrayToString( curTsBlock.getColumn(index).getBinary(tsBlockIndex).getValues()); - case TIMESTAMP: - return RpcUtils.formatDatetime( - timeFormat, "ms", curTsBlock.getColumn(index).getLong(tsBlockIndex), zoneId); case DATE: return DateUtils.formatDate(curTsBlock.getColumn(index).getInt(tsBlockIndex)); default: diff --git a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 index c2086e8de891..6d3ffe0b837d 100644 --- a/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 +++ b/iotdb-core/antlr/src/main/antlr4/org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 @@ -82,7 +82,7 @@ dclStatement ; utilityStatement - : flush | clearCache | settle | startRepairData | stopRepairData | explain + : flush | clearCache | setConfiguration | settle | startRepairData | stopRepairData | explain | setSystemStatus | showVersion | showFlushInfo | showLockInfo | showQueryResource | showQueries | showCurrentTimestamp | killQuery | grantWatermarkEmbedding | revokeWatermarkEmbedding | loadConfiguration | loadTimeseries | loadFile @@ -973,6 +973,15 @@ clearCache : CLEAR CACHE (ON (LOCAL | CLUSTER))? ; +// Set Configuration +setConfiguration + : SET CONFIGURATION setConfigurationEntry+ (ON INTEGER_LITERAL)? + ; + +setConfigurationEntry + : STRING_LITERAL OPERATOR_SEQ STRING_LITERAL + ; + // Settle settle : SETTLE (prefixPath|tsFilePath=STRING_LITERAL) diff --git a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat index e69764266887..8c0e392f71bd 100644 --- a/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat +++ b/iotdb-core/confignode/src/assembly/resources/sbin/stop-confignode.bat @@ -22,10 +22,15 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ -IF EXIST "%superior%\conf\iotdb-system.properties" ( +IF EXIST "%superior_dir%\conf\iotdb-system.properties" ( set config_file="%superior_dir%\conf\iotdb-system.properties" ) ELSE ( - set config_file="%superior_dir%\conf\iotdb-confignode.properties" + IF EXIST "%superior_dir%\conf\iotdb-confignode.properties" ( + set config_file="%superior_dir%\conf\iotdb-confignode.properties" + ) ELSE ( + echo "No configuration file found. Exiting." + exit /b 1 + ) ) for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "^cn_internal_port" @@ -33,6 +38,11 @@ for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "^cn_internal_port" set cn_internal_port=%%i ) +if not defined cn_internal_port ( + echo "cn_internal_port not found in the configuration file. Exiting." + exit /b 1 +) + echo "check whether the cn_internal_port is used..., port is %cn_internal_port%" for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "cn_internal_address" @@ -40,6 +50,11 @@ for /f "eol=; tokens=2,2 delims==" %%i in ('findstr /i "cn_internal_address" set cn_internal_address=%%i ) +if not defined cn_internal_address ( + echo "cn_internal_address not found in the configuration file. Exiting." + exit /b 1 +) + for /f "tokens=5" %%a in ('netstat /ano ^| findstr %cn_internal_address%:%cn_internal_port% ^| findstr LISTENING ') do ( taskkill /f /pid %%a echo "close ConfigNode, PID:" %%a diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java index e9aa1e93f9d7..5f3064b4eed7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/ConfigNodeRequestType.java @@ -27,5 +27,7 @@ public enum ConfigNodeRequestType { REMOVE_CONFIG_NODE, DELETE_CONFIG_NODE_PEER, REPORT_CONFIG_NODE_SHUTDOWN, - STOP_CONFIG_NODE + STOP_CONFIG_NODE, + SET_CONFIGURATION, + SHOW_CONFIGURATION, } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java index 398b0e3557b8..b53035b29d5a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/DataNodeRequestType.java @@ -32,6 +32,8 @@ public enum DataNodeRequestType { STOP_REPAIR_DATA, LOAD_CONFIGURATION, SET_SYSTEM_STATUS, + SET_CONFIGURATION, + SHOW_CONFIGURATION, // Region Maintenance CREATE_DATA_REGION, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java index fe7a7ebfad31..31bc4074701c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/AsyncDataNodeClientPool.java @@ -23,6 +23,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TFlushReq; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; @@ -342,6 +343,12 @@ private void sendAsyncRequestToDataNode( (AsyncTSStatusRPCHandler) clientHandler.createAsyncRPCHandler(requestId, targetDataNode)); break; + case SET_CONFIGURATION: + client.setConfiguration( + (TSetConfigurationReq) clientHandler.getRequest(requestId), + (AsyncTSStatusRPCHandler) + clientHandler.createAsyncRPCHandler(requestId, targetDataNode)); + break; case START_REPAIR_DATA: client.startRepairData( (AsyncTSStatusRPCHandler) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java index 04d2c0917698..ab85a9badcb9 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncConfigNodeClientPool.java @@ -22,6 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation; import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.commons.client.ClientPoolFactory; import org.apache.iotdb.commons.client.IClientManager; import org.apache.iotdb.commons.client.exception.ClientManagerException; @@ -88,6 +89,8 @@ public Object sendSyncRequestToConfigNodeWithRetry( case STOP_CONFIG_NODE: // Only use stopConfigNode when the ConfigNode is removed. return client.stopConfigNode((TConfigNodeLocation) req); + case SET_CONFIGURATION: + return client.setConfiguration((TSetConfigurationReq) req); default: return RpcUtils.getStatus( TSStatusCode.EXECUTE_STATEMENT_ERROR, "Unknown request type: " + requestType); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java index 67a018faa703..86afa20d6acc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/sync/SyncDataNodeClientPool.java @@ -65,7 +65,7 @@ private SyncDataNodeClientPool() { new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory()); } - public TSStatus sendSyncRequestToDataNodeWithRetry( + public Object sendSyncRequestToDataNodeWithRetry( TEndPoint endPoint, Object req, DataNodeRequestType requestType) { Throwable lastException = new TException(); for (int retry = 0; retry < DEFAULT_RETRY_NUM; retry++) { @@ -84,7 +84,7 @@ public TSStatus sendSyncRequestToDataNodeWithRetry( .setMessage("All retry failed due to: " + lastException.getMessage()); } - public TSStatus sendSyncRequestToDataNodeWithGivenRetry( + public Object sendSyncRequestToDataNodeWithGivenRetry( TEndPoint endPoint, Object req, DataNodeRequestType requestType, int retryNum) { Throwable lastException = new TException(); for (int retry = 0; retry < retryNum; retry++) { @@ -103,7 +103,7 @@ public TSStatus sendSyncRequestToDataNodeWithGivenRetry( .setMessage("All retry failed due to: " + lastException.getMessage()); } - private TSStatus executeSyncRequest( + private Object executeSyncRequest( DataNodeRequestType requestType, SyncDataNodeInternalServiceClient client, Object req) throws TException { switch (requestType) { @@ -141,6 +141,8 @@ private TSStatus executeSyncRequest( return client.deleteOldRegionPeer((TMaintainPeerReq) req); case RESET_PEER_LIST: return client.resetPeerList((TResetPeerListReq) req); + case SHOW_CONFIGURATION: + return client.showConfiguration(); default: return RpcUtils.getStatus( TSStatusCode.EXECUTE_STATEMENT_ERROR, "Unknown request type: " + requestType); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java index 94ed04849ecc..e73ab25debe4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java @@ -21,7 +21,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.commons.conf.ConfigFileAutoUpdateTool; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.schema.SchemaConstant; @@ -55,13 +55,13 @@ public class ConfigNodeDescriptor { private final ConfigNodeConfig conf = new ConfigNodeConfig(); static { - ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); URL configNodeUrl = getPropsUrl(CommonConfig.OLD_CONFIG_NODE_CONFIG_NAME); URL dataNodeUrl = getPropsUrl(CommonConfig.OLD_DATA_NODE_CONFIG_NAME); URL commonConfigUrl = getPropsUrl(CommonConfig.OLD_COMMON_CONFIG_NAME); try { - updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); + ConfigurationFileUtils.checkAndMayUpdate( + systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); } catch (Exception e) { LOGGER.error("Failed to update config file", e); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 5b6caf403af9..4a3a0f7cf3b3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -28,14 +28,17 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.common.rpc.thrift.TSchemaNode; import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.auth.AuthException; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; @@ -210,7 +213,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; +import java.net.URL; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -221,6 +226,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -1498,6 +1504,33 @@ public TSStatus clearCache() { : status; } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) { + TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + int currentNodeId = CONF.getConfigNodeId(); + if (req.getNodeId() < 0 || currentNodeId == req.getNodeId()) { + URL url = ConfigNodeDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (url == null || !new File(url.getFile()).exists()) { + return tsStatus; + } + File file = new File(url.getFile()); + Properties properties = new Properties(); + properties.putAll(req.getConfigs()); + try { + ConfigurationFileUtils.updateConfigurationFile(file, properties); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + if (CONF.getConfigNodeId() == req.getNodeId()) { + return tsStatus; + } + } + tsStatus = confirmLeader(); + return tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? RpcUtils.squashResponseStatusList(nodeManager.setConfiguration(req)) + : tsStatus; + } + @Override public TSStatus startRepairData() { TSStatus status = confirmLeader(); @@ -1522,6 +1555,25 @@ public TSStatus loadConfiguration() { : status; } + @Override + public TShowConfigurationResp showConfiguration(int nodeId) { + if (ConfigNodeDescriptor.getInstance().getConf().getConfigNodeId() == nodeId) { + TShowConfigurationResp resp = + new TShowConfigurationResp(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS), ""); + try { + URL propsUrl = ConfigNodeDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + resp.setContent(ConfigurationFileUtils.readConfigFileContent(propsUrl)); + } catch (Exception e) { + resp.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage())); + } + return resp; + } + TSStatus status = confirmLeader(); + return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() + ? nodeManager.showConfiguration(nodeId) + : new TShowConfigurationResp(status, ""); + } + @Override public TSStatus setSystemStatus(String systemStatus) { TSStatus status = confirmLeader(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 3ac0ee6ade41..919e876bac21 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -23,7 +23,9 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; @@ -479,6 +481,9 @@ TDataPartitionTableResp getOrCreateDataPartition( /** Clear cache on all DataNodes. */ TSStatus clearCache(); + /** Set Configuration. */ + TSStatus setConfiguration(TSetConfigurationReq req); + /** Check and repair unsorted tsfile by compaction. */ TSStatus startRepairData(); @@ -488,6 +493,9 @@ TDataPartitionTableResp getOrCreateDataPartition( /** Load configuration on all DataNodes. */ TSStatus loadConfiguration(); + /** Show content of configuration file on specified node */ + TShowConfigurationResp showConfiguration(int nodeId); + /** Set system status on all DataNodes. */ TSStatus setSystemStatus(String status); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java index 664fd6b2cd5d..a0af5578b59b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java @@ -26,6 +26,8 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.cluster.NodeType; import org.apache.iotdb.commons.cluster.RegionRoleType; @@ -33,9 +35,11 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.service.metric.MetricService; +import org.apache.iotdb.confignode.client.ConfigNodeRequestType; import org.apache.iotdb.confignode.client.DataNodeRequestType; import org.apache.iotdb.confignode.client.async.AsyncDataNodeClientPool; import org.apache.iotdb.confignode.client.async.handlers.AsyncClientHandler; +import org.apache.iotdb.confignode.client.sync.SyncConfigNodeClientPool; import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool; import org.apache.iotdb.confignode.conf.ConfigNodeConfig; import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor; @@ -727,6 +731,54 @@ public List clearCache() { return clientHandler.getResponseList(); } + public List setConfiguration(TSetConfigurationReq req) { + List responseList = new ArrayList<>(); + + Map dataNodeLocationMap = + configManager.getNodeManager().getRegisteredDataNodeLocations(); + Map targetDataNodes = new HashMap<>(); + int nodeId = req.getNodeId(); + // send to datanode + if (dataNodeLocationMap.containsKey(nodeId)) { + targetDataNodes.put(nodeId, dataNodeLocationMap.get(nodeId)); + } else if (nodeId < 0) { + targetDataNodes.putAll(dataNodeLocationMap); + } + if (!targetDataNodes.isEmpty()) { + AsyncClientHandler clientHandler = + new AsyncClientHandler<>(DataNodeRequestType.SET_CONFIGURATION, req, dataNodeLocationMap); + AsyncDataNodeClientPool.getInstance().sendAsyncRequestToDataNodeWithRetry(clientHandler); + responseList.addAll(clientHandler.getResponseList()); + } + + // send to config node + List configNodes = getRegisteredConfigNodes(); + for (TConfigNodeLocation configNode : configNodes) { + if (configNode.getConfigNodeId() == CONF.getConfigNodeId()) { + continue; + } + if (nodeId >= 0 && nodeId != configNode.getConfigNodeId()) { + continue; + } + TSStatus status = null; + try { + status = + (TSStatus) + SyncConfigNodeClientPool.getInstance() + .sendSyncRequestToConfigNodeWithRetry( + configNode.getInternalEndPoint(), + new TSetConfigurationReq(req.getConfigs(), configNode.getConfigNodeId()), + ConfigNodeRequestType.SET_CONFIGURATION); + } catch (Exception e) { + status = + RpcUtils.getStatus( + TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode(), e.getMessage()); + } + responseList.add(status); + } + return responseList; + } + public List startRpairData() { Map dataNodeLocationMap = configManager.getNodeManager().getRegisteredDataNodeLocations(); @@ -754,6 +806,39 @@ public List loadConfiguration() { return clientHandler.getResponseList(); } + public TShowConfigurationResp showConfiguration(int nodeId) { + TShowConfigurationResp resp = new TShowConfigurationResp(); + + // data node + Map dataNodeLocationMap = + configManager.getNodeManager().getRegisteredDataNodeLocations(); + if (dataNodeLocationMap.containsKey(nodeId)) { + TDataNodeLocation dataNodeLocation = dataNodeLocationMap.get(nodeId); + return (TShowConfigurationResp) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeLocation.getInternalEndPoint(), + null, + DataNodeRequestType.SHOW_CONFIGURATION); + } + + // other config node + for (TConfigNodeLocation registeredConfigNode : getRegisteredConfigNodes()) { + if (registeredConfigNode.getConfigNodeId() != nodeId) { + continue; + } + resp = + (TShowConfigurationResp) + SyncConfigNodeClientPool.getInstance() + .sendSyncRequestToConfigNodeWithRetry( + registeredConfigNode.getInternalEndPoint(), + nodeId, + ConfigNodeRequestType.SHOW_CONFIGURATION); + return resp; + } + return resp; + } + public List setSystemStatus(String status) { Map dataNodeLocationMap = configManager.getNodeManager().getRegisteredDataNodeLocations(); @@ -765,11 +850,12 @@ public List setSystemStatus(String status) { } public TSStatus setDataNodeStatus(TSetDataNodeStatusReq setDataNodeStatusReq) { - return SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - setDataNodeStatusReq.getTargetDataNode().getInternalEndPoint(), - setDataNodeStatusReq.getStatus(), - DataNodeRequestType.SET_SYSTEM_STATUS); + return (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + setDataNodeStatusReq.getTargetDataNode().getInternalEndPoint(), + setDataNodeStatusReq.getStatus(), + DataNodeRequestType.SET_SYSTEM_STATUS); } /** @@ -802,11 +888,12 @@ private TSStatus killSpecificQuery(String queryId, TDataNodeLocation dataNodeLoc .setMessage( "The target DataNode is not existed, please ensure your input is correct"); } else { - return SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - dataNodeLocation.getInternalEndPoint(), - queryId, - DataNodeRequestType.KILL_QUERY_INSTANCE); + return (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeLocation.getInternalEndPoint(), + queryId, + DataNodeRequestType.KILL_QUERY_INSTANCE); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java index 35d82a5454ed..b16314ff5ad3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/partition/PartitionManager.java @@ -466,7 +466,7 @@ private TSStatus consensusWritePartitionResult(ConfigPhysicalPlan plan) { return getConsensusManager().write(plan); } catch (ConsensusException e) { // The allocation might fail due to consensus error - LOGGER.error("Write DataPartition allocation result failed because: {}", status); + LOGGER.error("Write partition allocation result failed because: {}", status); TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); res.setMessage(e.getMessage()); return res; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipePluginInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipePluginInfo.java index 382ebd51620e..d3c46c2957d7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipePluginInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipePluginInfo.java @@ -91,7 +91,8 @@ public void releasePipePluginInfoLock() { /////////////////////////////// Validator /////////////////////////////// - public void validateBeforeCreatingPipePlugin(String pluginName, String jarName, String jarMD5) { + public void validateBeforeCreatingPipePlugin( + final String pluginName, final String jarName, final String jarMD5) { // both build-in and user defined pipe plugin should be unique if (pipePluginMetaKeeper.containsPipePlugin(pluginName)) { throw new PipeException( @@ -108,9 +109,13 @@ public void validateBeforeCreatingPipePlugin(String pluginName, String jarName, } } - public void validateBeforeDroppingPipePlugin(String pluginName) { - if (pipePluginMetaKeeper.containsPipePlugin(pluginName) - && pipePluginMetaKeeper.getPipePluginMeta(pluginName).isBuiltin()) { + public void validateBeforeDroppingPipePlugin(final String pluginName) { + if (!pipePluginMetaKeeper.containsPipePlugin(pluginName)) { + throw new PipeException( + String.format( + "Failed to drop PipePlugin [%s], this PipePlugin has not been created", pluginName)); + } + if (pipePluginMetaKeeper.getPipePluginMeta(pluginName).isBuiltin()) { throw new PipeException( String.format( "Failed to drop PipePlugin [%s], the PipePlugin is a built-in PipePlugin", @@ -118,14 +123,14 @@ public void validateBeforeDroppingPipePlugin(String pluginName) { } } - public boolean isJarNeededToBeSavedWhenCreatingPipePlugin(String jarName) { + public boolean isJarNeededToBeSavedWhenCreatingPipePlugin(final String jarName) { return !pipePluginMetaKeeper.containsJar(jarName); } public void checkPipePluginExistence( - Map extractorAttributes, - Map processorAttributes, - Map connectorAttributes) { + final Map extractorAttributes, + final Map processorAttributes, + final Map connectorAttributes) { final PipeParameters extractorParameters = new PipeParameters(extractorAttributes); final String extractorPluginName = extractorParameters.getStringOrDefault( @@ -170,7 +175,7 @@ public void checkPipePluginExistence( /////////////////////////////// Pipe Plugin Management /////////////////////////////// - public TSStatus createPipePlugin(CreatePipePluginPlan createPipePluginPlan) { + public TSStatus createPipePlugin(final CreatePipePluginPlan createPipePluginPlan) { try { final PipePluginMeta pipePluginMeta = createPipePluginPlan.getPipePluginMeta(); @@ -188,7 +193,7 @@ public TSStatus createPipePlugin(CreatePipePluginPlan createPipePluginPlan) { } return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); - } catch (Exception e) { + } catch (final Exception e) { final String errorMessage = String.format( "Failed to execute createPipePlugin(%s) on config nodes, because of %s", @@ -199,7 +204,7 @@ public TSStatus createPipePlugin(CreatePipePluginPlan createPipePluginPlan) { } } - public TSStatus dropPipePlugin(DropPipePluginPlan dropPipePluginPlan) { + public TSStatus dropPipePlugin(final DropPipePluginPlan dropPipePluginPlan) { final String pluginName = dropPipePluginPlan.getPluginName(); if (pipePluginMetaKeeper.containsPipePlugin(pluginName)) { @@ -217,17 +222,17 @@ public DataSet showPipePlugins() { Arrays.asList(pipePluginMetaKeeper.getAllPipePluginMeta())); } - public JarResp getPipePluginJar(GetPipePluginJarPlan getPipePluginJarPlan) { + public JarResp getPipePluginJar(final GetPipePluginJarPlan getPipePluginJarPlan) { try { final List jarList = new ArrayList<>(); - for (String jarName : getPipePluginJarPlan.getJarNames()) { + for (final String jarName : getPipePluginJarPlan.getJarNames()) { jarList.add( ExecutableManager.transferToBytebuffer( PipePluginExecutableManager.getInstance() .getFileStringUnderInstallByName(jarName))); } return new JarResp(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), jarList); - } catch (Exception e) { + } catch (final Exception e) { LOGGER.error("Get PipePlugin_Jar failed", e); return new JarResp( new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()) @@ -239,7 +244,7 @@ public JarResp getPipePluginJar(GetPipePluginJarPlan getPipePluginJarPlan) { /////////////////////////////// Snapshot Processor /////////////////////////////// @Override - public boolean processTakeSnapshot(File snapshotDir) throws IOException { + public boolean processTakeSnapshot(final File snapshotDir) throws IOException { acquirePipePluginInfoLock(); try { final File snapshotFile = new File(snapshotDir, SNAPSHOT_FILE_NAME); @@ -261,7 +266,7 @@ public boolean processTakeSnapshot(File snapshotDir) throws IOException { } @Override - public void processLoadSnapshot(File snapshotDir) throws IOException { + public void processLoadSnapshot(final File snapshotDir) throws IOException { acquirePipePluginInfoLock(); try { final File snapshotFile = new File(snapshotDir, SNAPSHOT_FILE_NAME); @@ -288,7 +293,7 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java index 1e2a7d7f1501..00d9569c6593 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java @@ -181,18 +181,20 @@ public boolean invalidateCache(String storageGroupName) throws IOException, TExc if (nodeStatus == NodeStatus.Running) { // Always invalidate PartitionCache first final TSStatus invalidatePartitionStatus = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - dataNodeConfiguration.getLocation().getInternalEndPoint(), - invalidateCacheReq, - DataNodeRequestType.INVALIDATE_PARTITION_CACHE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeConfiguration.getLocation().getInternalEndPoint(), + invalidateCacheReq, + DataNodeRequestType.INVALIDATE_PARTITION_CACHE); final TSStatus invalidateSchemaStatus = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - dataNodeConfiguration.getLocation().getInternalEndPoint(), - invalidateCacheReq, - DataNodeRequestType.INVALIDATE_SCHEMA_CACHE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + dataNodeConfiguration.getLocation().getInternalEndPoint(), + invalidateCacheReq, + DataNodeRequestType.INVALIDATE_SCHEMA_CACHE); if (!verifySucceed(invalidatePartitionStatus, invalidateSchemaStatus)) { LOG.error( diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java index ee3f8855d336..4f82d5c502fb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java @@ -140,11 +140,12 @@ public void broadcastDisableDataNode(TDataNodeLocation disabledDataNode) { for (TDataNodeConfiguration node : otherOnlineDataNodes) { TDisableDataNodeReq disableReq = new TDisableDataNodeReq(disabledDataNode); TSStatus status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - node.getLocation().getInternalEndPoint(), - disableReq, - DataNodeRequestType.DISABLE_DATA_NODE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + node.getLocation().getInternalEndPoint(), + disableReq, + DataNodeRequestType.DISABLE_DATA_NODE); if (!isSucceed(status)) { LOGGER.error( "{}, BroadcastDisableDataNode meets error, disabledDataNode: {}, error: {}", @@ -224,11 +225,12 @@ public TSStatus createNewRegionPeer(TConsensusGroupId regionId, TDataNodeLocatio TCreatePeerReq req = new TCreatePeerReq(regionId, currentPeerNodes, storageGroup); status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - destDataNode.getInternalEndPoint(), - req, - DataNodeRequestType.CREATE_NEW_REGION_PEER); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + destDataNode.getInternalEndPoint(), + req, + DataNodeRequestType.CREATE_NEW_REGION_PEER); if (isSucceed(status)) { LOGGER.info( @@ -269,11 +271,12 @@ public TSStatus submitAddRegionPeerTask( // destDataNode is where the new RegionReplica is created TMaintainPeerReq maintainPeerReq = new TMaintainPeerReq(regionId, destDataNode, procedureId); status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - coordinator.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.ADD_REGION_PEER); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + coordinator.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.ADD_REGION_PEER); LOGGER.info( "{}, Send action addRegionPeer finished, regionId: {}, rpcDataNode: {}, destDataNode: {}, status: {}", REGION_MIGRATE_PROCESS, @@ -305,11 +308,12 @@ public TSStatus submitRemoveRegionPeerTask( TMaintainPeerReq maintainPeerReq = new TMaintainPeerReq(regionId, originalDataNode, procedureId); status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - coordinator.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.REMOVE_REGION_PEER); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + coordinator.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.REMOVE_REGION_PEER); LOGGER.info( "{}, Send action removeRegionPeer finished, regionId: {}, rpcDataNode: {}", REGION_MIGRATE_PROCESS, @@ -338,17 +342,19 @@ public TSStatus submitDeleteOldRegionPeerTask( status = configManager.getLoadManager().getNodeStatus(originalDataNode.getDataNodeId()) == NodeStatus.Unknown - ? SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithGivenRetry( - originalDataNode.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.DELETE_OLD_REGION_PEER, - 1) - : SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - originalDataNode.getInternalEndPoint(), - maintainPeerReq, - DataNodeRequestType.DELETE_OLD_REGION_PEER); + ? (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithGivenRetry( + originalDataNode.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.DELETE_OLD_REGION_PEER, + 1) + : (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + originalDataNode.getInternalEndPoint(), + maintainPeerReq, + DataNodeRequestType.DELETE_OLD_REGION_PEER); LOGGER.info( "{}, Send action deleteOldRegionPeer finished, regionId: {}, dataNodeId: {}", REGION_MIGRATE_PROCESS, @@ -499,9 +505,13 @@ public void stopDataNode(TDataNodeLocation dataNode) { REMOVE_DATANODE_PROCESS, dataNode); TSStatus status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithGivenRetry( - dataNode.getInternalEndPoint(), dataNode, DataNodeRequestType.STOP_DATA_NODE, 2); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithGivenRetry( + dataNode.getInternalEndPoint(), + dataNode, + DataNodeRequestType.STOP_DATA_NODE, + 2); configManager.getLoadManager().removeNodeCache(dataNode.getDataNodeId()); LOGGER.info( "{}, Stop Data Node result: {}, stoppedDataNode: {}", diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java index b3f10181c201..badc5d2aa96b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/sync/AuthOperationProcedure.java @@ -107,11 +107,12 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, AuthOperationProcedu continue; } status = - SyncDataNodeClientPool.getInstance() - .sendSyncRequestToDataNodeWithRetry( - pair.getLeft().getLocation().getInternalEndPoint(), - req, - DataNodeRequestType.INVALIDATE_PERMISSION_CACHE); + (TSStatus) + SyncDataNodeClientPool.getInstance() + .sendSyncRequestToDataNodeWithRetry( + pair.getLeft().getLocation().getInternalEndPoint(), + req, + DataNodeRequestType.INVALIDATE_PERMISSION_CACHE); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { it.remove(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 493bffa5712a..a237c57d62c2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -23,9 +23,11 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.path.PartialPath; @@ -808,6 +810,11 @@ public TSStatus clearCache() { return configManager.clearCache(); } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) throws TException { + return configManager.setConfiguration(req); + } + @Override public TSStatus startRepairData() { return configManager.startRepairData(); @@ -823,6 +830,11 @@ public TSStatus loadConfiguration() { return configManager.loadConfiguration(); } + @Override + public TShowConfigurationResp showConfiguration(int nodeId) throws TException { + return configManager.showConfiguration(nodeId); + } + @Override public TSStatus setSystemStatus(String status) { return configManager.setSystemStatus(status); diff --git a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat index d249d4275674..fec692412346 100644 --- a/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat +++ b/iotdb-core/datanode/src/assembly/resources/sbin/stop-datanode.bat @@ -22,10 +22,20 @@ set current_dir=%~dp0 set superior_dir=%current_dir%\..\ -IF EXIST "%superior%\conf\iotdb-system.properties" ( +IF EXIST "%superior_dir%\conf\iotdb-system.properties" ( set config_file="%superior_dir%\conf\iotdb-system.properties" ) ELSE ( - set config_file="%superior_dir%\conf\iotdb-datanode.properties" + IF EXIST "%superior_dir%\conf\iotdb-datanode.properties" ( + set config_file=%superior_dir%\conf\iotdb-datanode.properties + ) ELSE ( + echo No configuration file found. Exiting. + exit /b 1 + ) +) + +if not defined config_file ( + echo No configuration file found. Exiting. + exit /b 1 ) for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" @@ -33,6 +43,11 @@ for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "^dn_rpc_port" set dn_rpc_port=%%i ) +if not defined dn_rpc_port ( + echo dn_rpc_port not found in the configuration file. Exiting. + exit /b 1 +) + echo Check whether the rpc_port is used..., port is %dn_rpc_port% for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "dn_rpc_address" @@ -40,6 +55,11 @@ for /f "eol=# tokens=2 delims==" %%i in ('findstr /i "dn_rpc_address" set dn_rpc_address=%%i ) +if not defined dn_rpc_address ( + echo dn_rpc_address not found in the configuration file. Exiting. + exit /b 1 +) + for /f "tokens=5" %%a in ('netstat /ano ^| findstr %dn_rpc_address%:%dn_rpc_port%') do ( taskkill /f /pid %%a echo Close DataNode, PID: %%a diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java index 43ab611e54b8..bb5f557ec0ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/audit/AuditLogger.java @@ -228,6 +228,7 @@ private static AuditLogOperation judgeLogOperation(StatementType type) { case INTERNAL_CREATE_TIMESERIES: case START_REPAIR_DATA: case STOP_REPAIR_DATA: + case SET_CONFIGURATION: return AuditLogOperation.DML; case LIST_USER: case LIST_ROLE: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index fd1068b12577..c54b7d5ef4f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -20,7 +20,7 @@ import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; -import org.apache.iotdb.commons.conf.ConfigFileAutoUpdateTool; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.exception.BadNodeUrlException; import org.apache.iotdb.commons.schema.SchemaConstant; @@ -101,13 +101,13 @@ public class IoTDBDescriptor { private static final double MIN_DIR_USE_PROPORTION = 0.5; static { - ConfigFileAutoUpdateTool updateTool = new ConfigFileAutoUpdateTool(); URL systemConfigUrl = getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); URL configNodeUrl = getPropsUrl(CommonConfig.OLD_CONFIG_NODE_CONFIG_NAME); URL dataNodeUrl = getPropsUrl(CommonConfig.OLD_DATA_NODE_CONFIG_NAME); URL commonConfigUrl = getPropsUrl(CommonConfig.OLD_COMMON_CONFIG_NAME); try { - updateTool.checkAndMayUpdate(systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); + ConfigurationFileUtils.checkAndMayUpdate( + systemConfigUrl, configNodeUrl, dataNodeUrl, commonConfigUrl); } catch (Exception e) { LOGGER.error("Failed to update config file", e); } @@ -1849,8 +1849,11 @@ public void loadHotModifiedProps() throws QueryProcessException { throw new QueryProcessException( String.format("Fail to reload config file %s because %s", url, e.getMessage())); } - ReloadLevel reloadLevel = - MetricConfigDescriptor.getInstance().loadHotProps(commonProperties, false); + reloadMetricProperties(commonProperties); + } + + public void reloadMetricProperties(Properties properties) { + ReloadLevel reloadLevel = MetricConfigDescriptor.getInstance().loadHotProps(properties, false); LOGGER.info("Reload metric service in level {}", reloadLevel); if (reloadLevel == ReloadLevel.RESTART_INTERNAL_REPORTER) { IoTDBInternalReporter internalReporter; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index 3cfe14d99084..4fff64feb2eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -24,9 +24,11 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.client.ClientManager; import org.apache.iotdb.commons.client.ThriftClient; import org.apache.iotdb.commons.client.factory.ThriftClientFactory; @@ -648,6 +650,12 @@ public TSStatus clearCache() throws TException { () -> client.clearCache(), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.setConfiguration(req), status -> !updateConfigNodeLeader(status)); + } + @Override public TSStatus startRepairData() throws TException { return executeRemoteCallWithRetry( @@ -666,6 +674,12 @@ public TSStatus loadConfiguration() throws TException { () -> client.loadConfiguration(), status -> !updateConfigNodeLeader(status)); } + @Override + public TShowConfigurationResp showConfiguration(int nodeId) throws TException { + return executeRemoteCallWithRetry( + () -> client.showConfiguration(nodeId), resp -> !updateConfigNodeLeader(resp.getStatus())); + } + @Override public TSStatus setSystemStatus(String systemStatus) throws TException { return executeRemoteCallWithRetry( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java index 0acddacb690f..b66b2dd4f359 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/ClientRPCServiceImpl.java @@ -23,8 +23,13 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationTemplateResp; import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; +import org.apache.iotdb.commons.client.exception.ClientManagerException; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.exception.IllegalPathException; @@ -46,6 +51,9 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.pipe.agent.PipeAgent; import org.apache.iotdb.db.protocol.basic.BasicOpenSessionResp; +import org.apache.iotdb.db.protocol.client.ConfigNodeClient; +import org.apache.iotdb.db.protocol.client.ConfigNodeClientManager; +import org.apache.iotdb.db.protocol.client.ConfigNodeInfo; import org.apache.iotdb.db.protocol.session.IClientSession; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.protocol.thrift.OperationType; @@ -2366,6 +2374,67 @@ public TSQueryTemplateResp querySchemaTemplate(TSQueryTemplateReq req) { } } + @Override + public TShowConfigurationTemplateResp showConfigurationTemplate() throws TException { + TShowConfigurationTemplateResp resp = new TShowConfigurationTemplateResp(); + try { + IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(clientSession)) { + resp.setStatus(getNotLoggedInStatus()); + return resp; + } + resp.setContent(ConfigurationFileUtils.readConfigurationTemplateFile()); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + return resp; + } catch (Exception e) { + resp.setStatus( + onNpeOrUnexpectedException( + e, OperationType.EXECUTE_QUERY_STATEMENT, TSStatusCode.EXECUTE_STATEMENT_ERROR)); + return resp; + } finally { + SESSION_MANAGER.updateIdleTime(); + } + } + + @Override + public TShowConfigurationResp showConfiguration(int nodeId) throws TException { + TShowConfigurationResp resp = new TShowConfigurationResp(); + resp.setContent(""); + try { + IClientSession clientSession = SESSION_MANAGER.getCurrSessionAndUpdateIdleTime(); + if (!SESSION_MANAGER.checkLogin(clientSession)) { + resp.setStatus(getNotLoggedInStatus()); + return resp; + } + if (!clientSession.getUsername().equals(AuthorityChecker.SUPER_USER)) { + resp.setStatus(RpcUtils.getStatus(TSStatusCode.NO_PERMISSION)); + return resp; + } + + if (IoTDBDescriptor.getInstance().getConfig().getDataNodeId() == nodeId) { + resp.setContent( + ConfigurationFileUtils.readConfigFileContent( + IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME))); + resp.setStatus(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS)); + return resp; + } + + try (ConfigNodeClient client = + ConfigNodeClientManager.getInstance().borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + return client.showConfiguration(nodeId); + } catch (ClientManagerException e) { + throw new TException(e); + } + } catch (Exception e) { + resp.setStatus( + onNpeOrUnexpectedException( + e, OperationType.EXECUTE_QUERY_STATEMENT, TSStatusCode.EXECUTE_STATEMENT_ERROR)); + return resp; + } finally { + SESSION_MANAGER.updateIdleTime(); + } + } + private TSQueryTemplateResp executeTemplateQueryStatement( Statement statement, TSQueryTemplateReq req, TSQueryTemplateResp resp) { long startTime = System.nanoTime(); @@ -2739,7 +2808,7 @@ private TSExecuteStatementResp createResponse(DatasetHeader header, long queryId return resp; } - private TSStatus getNotLoggedInStatus() { + protected TSStatus getNotLoggedInStatus() { return RpcUtils.getStatus( TSStatusCode.NOT_LOGIN, "Log in failed. Either you are not authorized or the session has timed out."); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 0765044f5f00..174d91350d8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -24,13 +24,16 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSettleReq; +import org.apache.iotdb.common.rpc.thrift.TShowConfigurationResp; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant.ClientVersion; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.DataRegionId; @@ -250,6 +253,7 @@ import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; +import java.net.URL; import java.nio.ByteBuffer; import java.time.ZoneId; import java.util.ArrayList; @@ -1748,6 +1752,11 @@ public TSStatus clearCache() throws TException { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + @Override + public TSStatus setConfiguration(TSetConfigurationReq req) { + return StorageEngine.getInstance().setConfiguration(req); + } + @Override public TSStatus settle(TSettleReq req) throws TException { return SettleRequestHandler.getInstance().handleSettleRequest(req); @@ -1763,6 +1772,19 @@ public TSStatus loadConfiguration() throws TException { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + @Override + public TShowConfigurationResp showConfiguration() { + TShowConfigurationResp resp = + new TShowConfigurationResp(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS), ""); + try { + URL propsUrl = IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + resp.setContent(ConfigurationFileUtils.readConfigFileContent(propsUrl)); + } catch (Exception e) { + resp.setStatus(RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage())); + } + return resp; + } + @Override public TSStatus setSystemStatus(String status) throws TException { try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java index cb383edd45b1..1d42fafdef6a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/Analysis.java @@ -39,6 +39,8 @@ import org.apache.iotdb.db.queryengine.plan.expression.Expression; import org.apache.iotdb.db.queryengine.plan.expression.ExpressionType; import org.apache.iotdb.db.queryengine.plan.expression.leaf.TimeSeriesOperand; +import org.apache.iotdb.db.queryengine.plan.planner.plan.TimePredicate; +import org.apache.iotdb.db.queryengine.plan.planner.plan.TreeModelTimePredicate; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.FilterNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.DeviceViewIntoPathDescriptor; import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.FillDescriptor; @@ -378,6 +380,7 @@ public void setStatement(Statement statement) { this.statement = statement; } + @Override public DataPartition getDataPartitionInfo() { return dataPartition; } @@ -386,6 +389,7 @@ public void setDataPartitionInfo(DataPartition dataPartition) { this.dataPartition = dataPartition; } + @Override public SchemaPartition getSchemaPartitionInfo() { return schemaPartition; } @@ -406,10 +410,12 @@ public List getRedirectNodeList() { return redirectNodeList; } + @Override public void setRedirectNodeList(List redirectNodeList) { this.redirectNodeList = redirectNodeList; } + @Override public void addEndPointToRedirectNodeList(TEndPoint endPoint) { if (redirectNodeList == null) { redirectNodeList = new ArrayList<>(); @@ -425,6 +431,11 @@ public void setGlobalTimePredicate(Expression timeFilter) { this.globalTimePredicate = timeFilter; } + @Override + public TimePredicate getCovertedTimePredicate() { + return globalTimePredicate == null ? null : new TreeModelTimePredicate(globalTimePredicate); + } + @Override public DatasetHeader getRespDatasetHeader() { return respDatasetHeader; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java index 207dea7b9f3d..4adcd7e298ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/IAnalysis.java @@ -19,12 +19,18 @@ package org.apache.iotdb.db.queryengine.plan.analyze; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.partition.DataPartition; +import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; +import org.apache.iotdb.db.queryengine.plan.planner.plan.TimePredicate; import org.apache.tsfile.read.common.block.TsBlock; +import java.util.List; + public interface IAnalysis { boolean isFailed(); @@ -42,4 +48,14 @@ public interface IAnalysis { DatasetHeader getRespDatasetHeader(); String getStatementType(); + + SchemaPartition getSchemaPartitionInfo(); + + DataPartition getDataPartitionInfo(); + + void setRedirectNodeList(List redirectNodeList); + + void addEndPointToRedirectNodeList(TEndPoint endPoint); + + TimePredicate getCovertedTimePredicate(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java index a372772e3749..0d523238bca5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java @@ -82,6 +82,7 @@ import java.io.File; import java.io.IOException; +import java.nio.BufferUnderflowException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -177,6 +178,13 @@ public Analysis analyzeFileByFile() { String.format("TsFile %s is empty or incomplete.", tsFile.getPath())); } catch (AuthException e) { return createFailAnalysisForAuthException(e); + } catch (BufferUnderflowException e) { + LOGGER.warn( + "The file {} is not a valid tsfile. Please check the input file.", tsFile.getPath(), e); + throw new SemanticException( + String.format( + "The file %s is not a valid tsfile. Please check the input file.", + tsFile.getPath())); } catch (Exception e) { LOGGER.warn("Parse file {} to resource error.", tsFile.getPath(), e); throw new SemanticException( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java index b024ea5a62a3..0786d2fb8938 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/ConfigTaskVisitor.java @@ -70,6 +70,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.sys.KillQueryTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.LoadConfigurationTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.MergeTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetConfigurationTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.SetSystemStatusTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StartRepairDataTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.StopRepairDataTask; @@ -149,6 +150,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StartRepairDataStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.StopRepairDataStatement; @@ -259,6 +261,12 @@ public IConfigTask visitClearCache( return new ClearCacheTask(clearCacheStatement); } + @Override + public IConfigTask visitSetConfiguration( + SetConfigurationStatement setConfigurationStatement, MPPQueryContext context) { + return new SetConfigurationTask(setConfigurationStatement); + } + @Override public IConfigTask visitStartRepairData( StartRepairDataStatement startRepairDataStatement, MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index bb1ccc1aa017..63ae1cf2a20b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetSpaceQuotaReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.common.rpc.thrift.TSetThrottleQuotaReq; @@ -30,6 +31,7 @@ import org.apache.iotdb.commons.client.exception.ClientManagerException; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.consensus.ConfigRegionId; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.IoTDBException; @@ -485,7 +487,7 @@ public SettableFuture createFunction( } } catch (IOException | URISyntaxException e) { LOGGER.warn( - "Failed to get executable for UDF({}) using URI: {}, the cause is: {}", + "Failed to get executable for UDF({}) using URI: {}.", createFunctionStatement.getUdfName(), createFunctionStatement.getUriString(), e); @@ -521,14 +523,14 @@ public SettableFuture createFunction( | InvocationTargetException | ClassCastException e) { LOGGER.warn( - "Failed to create function when try to create UDF({}) instance first, the cause is: {}", + "Failed to create function when try to create UDF({}) instance first.", createFunctionStatement.getUdfName(), e); future.setException( new IoTDBException( "Failed to load class '" + createFunctionStatement.getClassName() - + "', because it's not found in jar file: " + + "', because it's not found in jar file or is invalid: " + createFunctionStatement.getUriString(), TSStatusCode.UDF_LOAD_CLASS_ERROR.getStatusCode())); return future; @@ -658,7 +660,7 @@ public SettableFuture createTrigger( } } catch (IOException | URISyntaxException e) { LOGGER.warn( - "Failed to get executable for Trigger({}) using URI: {}, the cause is: {}", + "Failed to get executable for Trigger({}) using URI: {}.", createTriggerStatement.getTriggerName(), createTriggerStatement.getUriString(), e); @@ -695,14 +697,14 @@ public SettableFuture createTrigger( | InvocationTargetException | ClassCastException e) { LOGGER.warn( - "Failed to create trigger when try to create trigger({}) instance first, the cause is: {}", + "Failed to create trigger when try to create trigger({}) instance first.", createTriggerStatement.getTriggerName(), e); future.setException( new IoTDBException( "Failed to load class '" + createTriggerStatement.getClassName() - + "', because it's not found in jar file: " + + "', because it's not found in jar file or is invalid: " + createTriggerStatement.getUriString(), TSStatusCode.TRIGGER_LOAD_CLASS_ERROR.getStatusCode())); return future; @@ -768,7 +770,7 @@ public SettableFuture showTriggers() { @Override public SettableFuture createPipePlugin( - CreatePipePluginStatement createPipePluginStatement) { + final CreatePipePluginStatement createPipePluginStatement) { final SettableFuture future = SettableFuture.create(); final String pluginName = createPipePluginStatement.getPluginName(); final String className = createPipePluginStatement.getClassName(); @@ -784,13 +786,13 @@ public SettableFuture createPipePlugin( try (final ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - String libRoot; - ByteBuffer jarFile; - String jarMd5; + final String libRoot; + final ByteBuffer jarFile; + final String jarMd5; final String jarFileName = new File(uriString).getName(); try { - URI uri = new URI(uriString); + final URI uri = new URI(uriString); if (uri.getScheme() == null) { future.setException( new IoTDBException( @@ -800,10 +802,10 @@ public SettableFuture createPipePlugin( } if (!uri.getScheme().equals("file")) { // Download executable - ExecutableResource resource = + final ExecutableResource resource = PipePluginExecutableManager.getInstance() .request(Collections.singletonList(uriString)); - String jarFilePathUnderTempDir = + final String jarFilePathUnderTempDir = PipePluginExecutableManager.getInstance() .getDirStringUnderTempRootByRequestId(resource.getRequestId()) + File.separator @@ -821,9 +823,9 @@ public SettableFuture createPipePlugin( // Set md5 of the jar file jarMd5 = DigestUtils.md5Hex(Files.newInputStream(Paths.get(libRoot))); } - } catch (IOException | URISyntaxException e) { + } catch (final IOException | URISyntaxException e) { LOGGER.warn( - "Failed to get executable for PipePlugin({}) using URI: {}, the cause is: {}", + "Failed to get executable for PipePlugin({}) using URI: {}.", createPipePluginStatement.getPluginName(), createPipePluginStatement.getUriString(), e); @@ -837,25 +839,26 @@ public SettableFuture createPipePlugin( } // try to create instance, this request will fail if creation is not successful - try (PipePluginClassLoader classLoader = new PipePluginClassLoader(libRoot)) { + try (final PipePluginClassLoader classLoader = new PipePluginClassLoader(libRoot)) { // ensure that jar file contains the class and the class is a pipe plugin - Class clazz = Class.forName(createPipePluginStatement.getClassName(), true, classLoader); - PipePlugin ignored = (PipePlugin) clazz.getDeclaredConstructor().newInstance(); - } catch (ClassNotFoundException + final Class clazz = + Class.forName(createPipePluginStatement.getClassName(), true, classLoader); + final PipePlugin ignored = (PipePlugin) clazz.getDeclaredConstructor().newInstance(); + } catch (final ClassNotFoundException | NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException | ClassCastException e) { LOGGER.warn( - "Failed to create function when try to create PipePlugin({}) instance first, the cause is: {}", + "Failed to create function when try to create PipePlugin({}) instance first.", createPipePluginStatement.getPluginName(), e); future.setException( new IoTDBException( "Failed to load class '" + createPipePluginStatement.getClassName() - + "', because it's not found in jar file: " + + "', because it's not found in jar file or is invalid: " + createPipePluginStatement.getUriString(), TSStatusCode.PIPE_PLUGIN_LOAD_CLASS_ERROR.getStatusCode())); return future; @@ -884,7 +887,7 @@ public SettableFuture createPipePlugin( } else { future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); } - } catch (ClientManagerException | TException | IOException e) { + } catch (final ClientManagerException | TException | IOException e) { future.setException(e); } return future; @@ -1040,6 +1043,46 @@ public SettableFuture clearCache(boolean onCluster) { return future; } + @Override + public SettableFuture setConfiguration(TSetConfigurationReq req) { + SettableFuture future = SettableFuture.create(); + + TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + List ignoredConfigItems = + ConfigurationFileUtils.filterImmutableConfigItems(req.getConfigs()); + TSStatus warningTsStatus = null; + if (!ignoredConfigItems.isEmpty()) { + warningTsStatus = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + warningTsStatus.setMessage("ignored config items: " + ignoredConfigItems); + if (req.getConfigs().isEmpty()) { + future.setException(new IoTDBException(warningTsStatus.message, warningTsStatus.code)); + return future; + } + } + + boolean onLocal = IoTDBDescriptor.getInstance().getConfig().getDataNodeId() == req.getNodeId(); + if (onLocal) { + tsStatus = StorageEngine.getInstance().setConfiguration(req); + } else { + try (ConfigNodeClient client = + CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + // Send request to some API server + tsStatus = client.setConfiguration(req); + } catch (ClientManagerException | TException e) { + future.setException(e); + } + } + if (warningTsStatus != null) { + tsStatus = warningTsStatus; + } + if (tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } else { + future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); + } + return future; + } + @Override public SettableFuture startRepairData(boolean onCluster) { SettableFuture future = SettableFuture.create(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index 39c4ed62166e..01716c2da9a8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.commons.cluster.NodeStatus; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; @@ -126,6 +127,8 @@ public interface IConfigTaskExecutor { SettableFuture clearCache(boolean onCluster); + SettableFuture setConfiguration(TSetConfigurationReq tSetConfigurationReq); + SettableFuture loadConfiguration(boolean onCluster); SettableFuture setSystemStatus(boolean onCluster, NodeStatus status); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/SetConfigurationTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/SetConfigurationTask.java new file mode 100644 index 000000000000..f03c0aed7bd5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/SetConfigurationTask.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.sys; + +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; + +import com.google.common.util.concurrent.ListenableFuture; + +public class SetConfigurationTask implements IConfigTask { + + private final SetConfigurationStatement setConfigurationStatement; + + public SetConfigurationTask(SetConfigurationStatement setConfigurationStatement) { + this.setConfigurationStatement = setConfigurationStatement; + } + + @Override + public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.setConfiguration( + new TSetConfigurationReq( + setConfigurationStatement.getConfigItems(), setConfigurationStatement.getNodeId())); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java index 12ddcf2636c7..261f3bb7ed93 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java @@ -196,6 +196,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.FlushStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; @@ -3246,6 +3247,25 @@ public Statement visitClearCache(IoTDBSqlParser.ClearCacheContext ctx) { return clearCacheStatement; } + // Set Configuration + + @Override + public Statement visitSetConfiguration(IoTDBSqlParser.SetConfigurationContext ctx) { + SetConfigurationStatement setConfigurationStatement = + new SetConfigurationStatement(StatementType.SET_CONFIGURATION); + int nodeId = + Integer.parseInt(ctx.INTEGER_LITERAL() == null ? "-1" : ctx.INTEGER_LITERAL().getText()); + Map configItems = new HashMap<>(); + for (IoTDBSqlParser.SetConfigurationEntryContext entry : ctx.setConfigurationEntry()) { + String key = entry.STRING_LITERAL(0).getText().replace("\"", ""); + String value = entry.STRING_LITERAL(1).getText().replace("\"", ""); + configItems.put(key, value); + } + setConfigurationStatement.setNodeId(nodeId); + setConfigurationStatement.setConfigItems(configItems); + return setConfigurationStatement; + } + // Start Repair Data @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/WriteFragmentParallelPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/WriteFragmentParallelPlanner.java index 6fb32419a4b6..b1d1fe801235 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/WriteFragmentParallelPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/distribution/WriteFragmentParallelPlanner.java @@ -21,13 +21,11 @@ import org.apache.iotdb.commons.partition.StorageExecutor; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; -import org.apache.iotdb.db.queryengine.plan.expression.Expression; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.IFragmentParallelPlaner; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.planner.plan.PlanFragment; import org.apache.iotdb.db.queryengine.plan.planner.plan.SubPlan; -import org.apache.iotdb.db.queryengine.plan.planner.plan.TreeModelTimePredicate; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; @@ -37,11 +35,11 @@ public class WriteFragmentParallelPlanner implements IFragmentParallelPlaner { private SubPlan subPlan; - private Analysis analysis; + private IAnalysis analysis; private MPPQueryContext queryContext; public WriteFragmentParallelPlanner( - SubPlan subPlan, Analysis analysis, MPPQueryContext queryContext) { + SubPlan subPlan, IAnalysis analysis, MPPQueryContext queryContext) { this.subPlan = subPlan; this.analysis = analysis; this.queryContext = queryContext; @@ -50,7 +48,6 @@ public WriteFragmentParallelPlanner( @Override public List parallelPlan() { PlanFragment fragment = subPlan.getPlanFragment(); - Expression globalTimePredicate = analysis.getGlobalTimePredicate(); PlanNode node = fragment.getPlanNodeTree(); if (!(node instanceof WritePlanNode)) { throw new IllegalArgumentException("PlanNode should be IWritePlanNode in WRITE operation"); @@ -62,7 +59,7 @@ public List parallelPlan() { new FragmentInstance( new PlanFragment(fragment.getId(), split), fragment.getId().genFragmentInstanceId(), - globalTimePredicate == null ? null : new TreeModelTimePredicate(globalTimePredicate), + analysis.getCovertedTimePredicate(), queryContext.getQueryType(), queryContext.getTimeOut(), queryContext.getSession()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/WritePlanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/WritePlanNode.java index 98a98e0ca196..43cbb20a40a4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/WritePlanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/WritePlanNode.java @@ -19,7 +19,7 @@ package org.apache.iotdb.db.queryengine.plan.planner.plan.node; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import java.util.List; @@ -29,5 +29,5 @@ protected WritePlanNode(PlanNodeId id) { super(id); } - public abstract List splitByPartition(Analysis analysis); + public abstract List splitByPartition(IAnalysis analysis); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java index 85bf2612510b..3623e9760076 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadSingleTsFileNode.java @@ -25,7 +25,7 @@ import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; @@ -188,7 +188,7 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { throw new NotImplementedException("split load single TsFile is not implemented"); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java index 74df1b08dbda..540f9e4d9f6d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFileNode.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.WritePlanNode; @@ -88,13 +89,13 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { List res = new ArrayList<>(); LoadTsFileStatement statement = - analysis.getStatement() instanceof PipeEnrichedStatement + ((Analysis) analysis).getStatement() instanceof PipeEnrichedStatement ? (LoadTsFileStatement) - ((PipeEnrichedStatement) analysis.getStatement()).getInnerStatement() - : (LoadTsFileStatement) analysis.getStatement(); + ((PipeEnrichedStatement) ((Analysis) analysis).getStatement()).getInnerStatement() + : (LoadTsFileStatement) ((Analysis) analysis).getStatement(); for (int i = 0; i < resources.size(); i++) { res.add( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java index 1f4e6e08a3d1..c3c4c3fa2e37 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/load/LoadTsFilePieceNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.db.queryengine.execution.load.TsFileData; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -147,7 +147,7 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { throw new NotImplementedException("split load piece TsFile is not implemented"); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/ActivateTemplateNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/ActivateTemplateNode.java index a8258a6ee01b..8749b93aca1e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/ActivateTemplateNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/ActivateTemplateNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -161,7 +161,7 @@ public static ActivateTemplateNode deserialize(ByteBuffer buffer) { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { TRegionReplicaSet regionReplicaSet = analysis.getSchemaPartitionInfo().getSchemaRegionReplicaSet(activatePath.getFullPath()); setRegionReplicaSet(regionReplicaSet); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/AlterTimeSeriesNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/AlterTimeSeriesNode.java index 9eef306f3c44..19984a441e8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/AlterTimeSeriesNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/AlterTimeSeriesNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -373,7 +373,7 @@ public void setRegionReplicaSet(TRegionReplicaSet regionReplicaSet) { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { TRegionReplicaSet regionReplicaSet = analysis.getSchemaPartitionInfo().getSchemaRegionReplicaSet(path.getIDeviceID().toString()); setRegionReplicaSet(regionReplicaSet); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/BatchActivateTemplateNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/BatchActivateTemplateNode.java index 8b71889d396f..bc0767e4c5f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/BatchActivateTemplateNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/BatchActivateTemplateNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -148,7 +148,7 @@ public static BatchActivateTemplateNode deserialize(ByteBuffer byteBuffer) { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { // gather devices to same target region Map>> splitMap = new HashMap<>(); for (Map.Entry> entry : templateActivationMap.entrySet()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateAlignedTimeSeriesNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateAlignedTimeSeriesNode.java index 600fab6b9a76..64e70aad1bdb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateAlignedTimeSeriesNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateAlignedTimeSeriesNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -455,7 +455,7 @@ public void setRegionReplicaSet(TRegionReplicaSet regionReplicaSet) { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { TRegionReplicaSet regionReplicaSet = analysis.getSchemaPartitionInfo().getSchemaRegionReplicaSet(devicePath.getFullPath()); setRegionReplicaSet(regionReplicaSet); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateMultiTimeSeriesNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateMultiTimeSeriesNode.java index 8fe92fc946fd..94d3e407fb58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateMultiTimeSeriesNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateMultiTimeSeriesNode.java @@ -23,7 +23,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -229,7 +229,7 @@ public void setRegionReplicaSet(TRegionReplicaSet regionReplicaSet) { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { // gather devices to same target region Map> splitMap = new HashMap<>(); for (Map.Entry entry : measurementGroupMap.entrySet()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTimeSeriesNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTimeSeriesNode.java index c455bc75dea4..8dcfccc41615 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTimeSeriesNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/CreateTimeSeriesNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.path.PartialPath; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -386,7 +386,7 @@ public TRegionReplicaSet getRegionReplicaSet() { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { TRegionReplicaSet regionReplicaSet = analysis.getSchemaPartitionInfo().getSchemaRegionReplicaSet(path.getIDeviceID().toString()); setRegionReplicaSet(regionReplicaSet); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalBatchActivateTemplateNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalBatchActivateTemplateNode.java index 6d78d8791349..66797b99939a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalBatchActivateTemplateNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalBatchActivateTemplateNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -141,7 +141,7 @@ public static InternalBatchActivateTemplateNode deserialize(ByteBuffer byteBuffe } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { // gather devices to same target region Map>> splitMap = new HashMap<>(); for (Map.Entry> entry : templateActivationMap.entrySet()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalCreateMultiTimeSeriesNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalCreateMultiTimeSeriesNode.java index b6e71de471f2..a15f8f92f144 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalCreateMultiTimeSeriesNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalCreateMultiTimeSeriesNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -143,7 +143,7 @@ public static InternalCreateMultiTimeSeriesNode deserialize(ByteBuffer byteBuffe } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { // gather devices to same target region Map>> splitMap = new HashMap<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalCreateTimeSeriesNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalCreateTimeSeriesNode.java index 99b11e6f501f..6caeb10a9e18 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalCreateTimeSeriesNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/InternalCreateTimeSeriesNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathDeserializeUtil; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -136,7 +136,7 @@ public static InternalCreateTimeSeriesNode deserialize(ByteBuffer byteBuffer) { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { TRegionReplicaSet regionReplicaSet = analysis.getSchemaPartitionInfo().getSchemaRegionReplicaSet(devicePath.getFullPath()); setRegionReplicaSet(regionReplicaSet); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/view/CreateLogicalViewNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/view/CreateLogicalViewNode.java index 28b8756858f8..01277fad540e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/view/CreateLogicalViewNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/metedata/write/view/CreateLogicalViewNode.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.path.PathDeserializeUtil; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -220,7 +220,7 @@ public static CreateLogicalViewNode deserialize(ByteBuffer byteBuffer) { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { Map> splitMap = new HashMap<>(); for (Map.Entry entry : this.viewPathToSourceMap.entrySet()) { // for each entry in the map for target path to source expression, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedDeleteDataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedDeleteDataNode.java index 4b5be108fb29..57450a3cd068 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedDeleteDataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedDeleteDataNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.db.consensus.statemachine.dataregion.DataExecutionVisitor; import org.apache.iotdb.db.queryengine.execution.executor.RegionWriteExecutor; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -162,7 +162,7 @@ public TRegionReplicaSet getRegionReplicaSet() { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { return deleteDataNode.splitByPartition(analysis).stream() .map( plan -> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedInsertNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedInsertNode.java index 596e771b21a4..855fe79de5d5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedInsertNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedInsertNode.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.db.consensus.statemachine.dataregion.DataExecutionVisitor; import org.apache.iotdb.db.queryengine.execution.executor.RegionWriteExecutor; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -134,7 +134,7 @@ public R accept(PlanVisitor visitor, C context) { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { return insertNode.splitByPartition(analysis).stream() .map( plan -> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedWritePlanNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedWritePlanNode.java index b83ac39b4b3f..2e7f7eeb4ee0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedWritePlanNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/pipe/PipeEnrichedWritePlanNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.db.consensus.statemachine.schemaregion.SchemaExecutionVisitor; import org.apache.iotdb.db.queryengine.execution.executor.RegionWriteExecutor; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -184,7 +184,7 @@ public TRegionReplicaSet getRegionReplicaSet() { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { return writePlanNode.splitByPartition(analysis).stream() .map( plan -> diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java index 877187be5a63..93eb2a873ab7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/DeleteDataNode.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.common.schematree.DeviceSchemaInfo; import org.apache.iotdb.db.queryengine.common.schematree.ISchemaTree; import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; @@ -281,8 +282,8 @@ public String toString() { } @Override - public List splitByPartition(Analysis analysis) { - ISchemaTree schemaTree = analysis.getSchemaTree(); + public List splitByPartition(IAnalysis analysis) { + ISchemaTree schemaTree = ((Analysis) analysis).getSchemaTree(); DataPartition dataPartition = analysis.getDataPartitionInfo(); Map> regionToPatternMap = new HashMap<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java index 683d6607ace1..430ab19c7c97 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertMultiTabletsNode.java @@ -22,7 +22,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.utils.StatusUtils; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -135,7 +135,7 @@ public void setSearchIndex(long index) { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { Map splitMap = new HashMap<>(); for (int i = 0; i < insertTabletNodeList.size(); i++) { InsertTabletNode insertTabletNode = insertTabletNodeList.get(i); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index d961f0d869c6..bda7132fb7bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -24,7 +24,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; @@ -107,7 +107,7 @@ public InsertRowNode( } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { TTimePartitionSlot timePartitionSlot = TimePartitionUtils.getTimePartitionSlot(time); this.dataRegionReplicaSet = analysis diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java index 215f38bb11f4..49fb7fac6318 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java @@ -25,7 +25,7 @@ import org.apache.iotdb.commons.consensus.index.ProgressIndex; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -232,7 +232,7 @@ public void markAsGeneratedByRemoteConsensusLeader() { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { Map splitMap = new HashMap<>(); List redirectInfo = new ArrayList<>(); for (int i = 0; i < insertRowNodeList.size(); i++) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java index 6fd53ce6da23..864880fd7749 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsOfOneDeviceNode.java @@ -26,7 +26,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; @@ -154,7 +154,7 @@ public List getOutputColumnNames() { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { List result = new ArrayList<>(); Map>> splitMap = new HashMap<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java index ed37df61e3f3..6fc61399b353 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertTabletNode.java @@ -25,7 +25,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; -import org.apache.iotdb.db.queryengine.plan.analyze.Analysis; +import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; @@ -192,7 +192,7 @@ public List getOutputColumnNames() { } @Override - public List splitByPartition(Analysis analysis) { + public List splitByPartition(IAnalysis analysis) { // only single device in single database List result = new ArrayList<>(); if (times.length == 0) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 9faf14e320d7..c4708ad19606 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -19,12 +19,15 @@ package org.apache.iotdb.db.queryengine.plan.relational.analyzer; +import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.partition.DataPartition; +import org.apache.iotdb.commons.partition.SchemaPartition; import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; import org.apache.iotdb.db.queryengine.plan.analyze.IAnalysis; import org.apache.iotdb.db.queryengine.plan.execution.memory.StatementMemorySource; +import org.apache.iotdb.db.queryengine.plan.planner.plan.TimePredicate; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; @@ -642,6 +645,31 @@ public String getStatementType() { return null; } + @Override + public SchemaPartition getSchemaPartitionInfo() { + throw new UnsupportedOperationException(); + } + + @Override + public DataPartition getDataPartitionInfo() { + return dataPartition; + } + + @Override + public void setRedirectNodeList(List redirectNodeList) { + throw new UnsupportedOperationException(); + } + + @Override + public void addEndPointToRedirectNodeList(TEndPoint endPoint) { + throw new UnsupportedOperationException(); + } + + @Override + public TimePredicate getCovertedTimePredicate() { + throw new UnsupportedOperationException(); + } + public static final class AccessControlInfo { private final AccessControl accessControl; private final Identity identity; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributionPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributionPlanner.java index 83f1548ee3a3..ccfd5b91cd0e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributionPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributionPlanner.java @@ -17,6 +17,7 @@ import org.apache.iotdb.db.queryengine.common.MPPQueryContext; import org.apache.iotdb.db.queryengine.execution.exchange.sink.DownStreamChannelLocation; import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.planner.distribution.WriteFragmentParallelPlanner; import org.apache.iotdb.db.queryengine.plan.planner.plan.DistributedQueryPlan; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; @@ -76,7 +77,9 @@ public DistributedQueryPlan plan() { subPlan.getPlanFragment().setRoot(true); List fragmentInstances = - new FragmentInstanceGenerator(subPlan, analysis, mppQueryContext).plan(); + mppQueryContext.getQueryType() == QueryType.READ + ? new TableModelQueryFragmentPlanner(subPlan, analysis, mppQueryContext).plan() + : new WriteFragmentParallelPlanner(subPlan, analysis, mppQueryContext).parallelPlan(); // Only execute this step for READ operation if (mppQueryContext.getQueryType() == QueryType.READ) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/FragmentInstanceGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelQueryFragmentPlanner.java similarity index 97% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/FragmentInstanceGenerator.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelQueryFragmentPlanner.java index 623217e340f4..2d9cf1a2b012 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/FragmentInstanceGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableModelQueryFragmentPlanner.java @@ -46,9 +46,10 @@ import java.util.List; import java.util.Map; -public class FragmentInstanceGenerator { +public class TableModelQueryFragmentPlanner { - private static final Logger LOGGER = LoggerFactory.getLogger(FragmentInstanceGenerator.class); + private static final Logger LOGGER = + LoggerFactory.getLogger(TableModelQueryFragmentPlanner.class); private final SubPlan subPlan; @@ -67,7 +68,7 @@ public class FragmentInstanceGenerator { // Record FragmentInstances dispatched to same DataNode private final Map> dataNodeFIMap = new HashMap<>(); - FragmentInstanceGenerator(SubPlan subPlan, Analysis analysis, MPPQueryContext queryContext) { + TableModelQueryFragmentPlanner(SubPlan subPlan, Analysis analysis, MPPQueryContext queryContext) { this.subPlan = subPlan; this.analysis = analysis; this.queryContext = queryContext; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java index 88906ba83f4c..4f49a5909e75 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementType.java @@ -184,4 +184,5 @@ public enum StatementType { SHOW_TOPICS, SHOW_SUBSCRIPTIONS, + SET_CONFIGURATION } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java index d8dba82f490b..248cf881de12 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/StatementVisitor.java @@ -112,6 +112,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.KillQueryStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.LoadConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.MergeStatement; +import org.apache.iotdb.db.queryengine.plan.statement.sys.SetConfigurationStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.SetSystemStatusStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowQueriesStatement; import org.apache.iotdb.db.queryengine.plan.statement.sys.ShowVersionStatement; @@ -391,6 +392,10 @@ public R visitClearCache(ClearCacheStatement clearCacheStatement, C context) { return visitStatement(clearCacheStatement, context); } + public R visitSetConfiguration(SetConfigurationStatement setConfigurationStatement, C context) { + return visitStatement(setConfigurationStatement, context); + } + public R visitStartRepairData(StartRepairDataStatement startRepairDataStatement, C context) { return visitStatement(startRepairDataStatement, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetConfigurationStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetConfigurationStatement.java new file mode 100644 index 000000000000..8f519de02807 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/sys/SetConfigurationStatement.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.statement.sys; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; +import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement; +import org.apache.iotdb.db.queryengine.plan.statement.Statement; +import org.apache.iotdb.db.queryengine.plan.statement.StatementType; +import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class SetConfigurationStatement extends Statement implements IConfigStatement { + + private int nodeId; + private Map configItems; + + public SetConfigurationStatement(StatementType setConfigurationType) { + this.statementType = setConfigurationType; + } + + public void setNodeId(int nodeId) { + this.nodeId = nodeId; + } + + public void setConfigItems(Map configItems) { + this.configItems = configItems; + } + + public int getNodeId() { + return nodeId; + } + + public Map getConfigItems() { + return configItems; + } + + @Override + public QueryType getQueryType() { + return QueryType.WRITE; + } + + @Override + public List getPaths() { + return Collections.emptyList(); + } + + @Override + public R accept(StatementVisitor visitor, C context) { + return visitor.visitSetConfiguration(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java index 0012b9936210..930d7d622a4b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/MultiInputLayer.java @@ -538,12 +538,11 @@ public YieldableState yield() throws Exception { } boolean findWindow = false; // Find target window or no more data to exit - while (!findWindow) { + while (!findWindow && cachedConsumed < cachedTimes.getPositionCount()) { while (cachedConsumed < cachedTimes.getPositionCount()) { long nextTime = cachedTimes.getLong(cachedConsumed); - if (curTime >= displayWindowEnd) { - nextIndexEnd--; + if (nextTime >= displayWindowEnd) { findWindow = true; break; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java index a57b9e108633..a419c71c535b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputMultiReferenceLayer.java @@ -428,12 +428,11 @@ public YieldableState yield() throws Exception { } boolean findWindow = false; // Find target window or no more data to exit - while (!findWindow) { + while (!findWindow && cachedConsumed < cachedTimes.getPositionCount()) { while (cachedConsumed < cachedTimes.getPositionCount()) { long nextTime = cachedTimes.getLong(cachedConsumed); - if (curTime >= displayWindowEnd) { - nextIndexEnd--; + if (nextTime >= displayWindowEnd) { findWindow = true; break; } @@ -591,19 +590,17 @@ public YieldableState yield() throws Exception { } // Set nextIndexEnd - long curTime = cachedTimes.getLong(cachedConsumed); if (cachedConsumed < cachedTimes.getPositionCount()) { nextIndexEnd++; cachedConsumed++; } boolean findWindow = false; // Find target window or no more data to exit - while (!findWindow) { + while (!findWindow && cachedConsumed < cachedTimes.getPositionCount()) { while (cachedConsumed < cachedTimes.getPositionCount()) { long nextTime = cachedTimes.getLong(cachedConsumed); - if (curTime >= displayWindowEnd) { - nextIndexEnd--; + if (nextTime >= displayWindowEnd) { findWindow = true; break; } @@ -615,7 +612,6 @@ public YieldableState yield() throws Exception { } nextIndexEnd++; cachedConsumed++; - curTime = nextTime; } if (!findWindow) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java index f7e6d93f8435..fdb0779867dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/intermediate/SingleInputSingleReferenceLayer.java @@ -352,12 +352,11 @@ public YieldableState yield() throws Exception { } boolean findWindow = false; // Find target window or no more data to exit - while (!findWindow) { + while (!findWindow && cachedConsumed < cachedTimes.getPositionCount()) { while (cachedConsumed < cachedTimes.getPositionCount()) { long nextTime = cachedTimes.getLong(cachedConsumed); - if (curTime >= displayWindowEnd) { - nextIndexEnd--; + if (nextTime >= displayWindowEnd) { findWindow = true; break; } @@ -516,19 +515,17 @@ public YieldableState yield() throws Exception { } // Set nextIndexEnd - long curTime = cachedTimes.getLong(cachedConsumed); if (cachedConsumed < cachedTimes.getPositionCount()) { nextIndexEnd++; cachedConsumed++; } boolean findWindow = false; // Find target window or no more data to exit - while (!findWindow) { + while (!findWindow && cachedConsumed < cachedTimes.getPositionCount()) { while (cachedConsumed < cachedTimes.getPositionCount()) { long nextTime = cachedTimes.getLong(cachedConsumed); - if (curTime >= displayWindowEnd) { - nextIndexEnd--; + if (nextTime >= displayWindowEnd) { findWindow = true; break; } @@ -540,7 +537,6 @@ public YieldableState yield() throws Exception { } nextIndexEnd++; cachedConsumed++; - curTime = nextTime; } if (!findWindow) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java index e0455b3329ff..d26304e113cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/StorageEngine.java @@ -20,12 +20,15 @@ import org.apache.iotdb.common.rpc.thrift.TFlushReq; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.common.rpc.thrift.TSetConfigurationReq; import org.apache.iotdb.common.rpc.thrift.TSetTTLReq; import org.apache.iotdb.commons.concurrent.ExceptionalCountDownLatch; import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory; import org.apache.iotdb.commons.concurrent.ThreadName; import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil; +import org.apache.iotdb.commons.conf.CommonConfig; import org.apache.iotdb.commons.conf.CommonDescriptor; +import org.apache.iotdb.commons.conf.ConfigurationFileUtils; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.commons.consensus.DataRegionId; import org.apache.iotdb.commons.consensus.index.ProgressIndex; @@ -83,12 +86,14 @@ import java.io.File; import java.io.IOException; +import java.net.URL; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; @@ -602,6 +607,43 @@ public void clearCache() { BloomFilterCache.getInstance().clear(); } + public TSStatus setConfiguration(TSetConfigurationReq req) { + TSStatus tsStatus = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); + Map newConfigItems = req.getConfigs(); + if (newConfigItems.isEmpty()) { + return tsStatus; + } + Properties properties = new Properties(); + properties.putAll(newConfigItems); + + URL configFileUrl = IoTDBDescriptor.getPropsUrl(CommonConfig.SYSTEM_CONFIG_NAME); + if (configFileUrl == null || !(new File(configFileUrl.getFile()).exists())) { + // configuration file not exist, update in mem + try { + IoTDBDescriptor.getInstance().loadHotModifiedProps(properties); + IoTDBDescriptor.getInstance().reloadMetricProperties(properties); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + return tsStatus; + } + + // 1. append new configuration properties to configuration file + try { + ConfigurationFileUtils.updateConfigurationFile(new File(configFileUrl.getFile()), properties); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + + // 2. load hot modified properties + try { + IoTDBDescriptor.getInstance().loadHotModifiedProps(); + } catch (Exception e) { + return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage()); + } + return tsStatus; + } + /** * Add a listener to listen flush start/end events. Notice that this addition only applies to * TsFileProcessors created afterwards. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java index 90e3c94305b1..1a3c394af7de 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/DateTimeUtils.java @@ -54,6 +54,8 @@ private DateTimeUtils() { // forbidding instantiation } + private static final String TIMESTAMP_PRECISION = + CommonDescriptor.getInstance().getConfig().getTimestampPrecision(); private static Function CAST_TIMESTAMP_TO_MS; static { @@ -825,7 +827,7 @@ public static Long parseDateTimeExpressionToLong(String dateExpression, ZoneId z parser1.getInterpreter().setPredictionMode(PredictionMode.SLL); parser1.removeErrorListeners(); parser1.addErrorListener(SqlParseError.INSTANCE); - return astVisitor.parseDateExpression(parser1.dateExpression(), "ms"); + return astVisitor.parseDateExpression(parser1.dateExpression(), TIMESTAMP_PRECISION); } public static Integer parseDateExpressionToInt(String dateExpression) { diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml index 421f791facec..5e5a1d251dc6 100644 --- a/iotdb-core/node-commons/pom.xml +++ b/iotdb-core/node-commons/pom.xml @@ -176,6 +176,15 @@ + + + ${project.basedir}/src/assembly/resources/conf + + iotdb-system.properties + + false + + + + generate-config-unix + compile + + exec + + + sh + + ${project.basedir}/src/assembly/resources/conf/generate_properties.sh + + + + + + + + + + generate-properties-windows + + + windows + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + + + + generate-properties-windows + compile + + exec + + + cmd + + /c + ${project.basedir}\src\assembly\resources\conf\generate_properties.bat + + + + + + + + diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat new file mode 100644 index 000000000000..f158d7fe6d2f --- /dev/null +++ b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.bat @@ -0,0 +1,64 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM +@echo off +setlocal enabledelayedexpansion + +set "source_file=src\assembly\resources\conf\iotdb-system.properties" +set "target_template_file=target\conf\iotdb-system.properties.template" +set "target_properties_file=target\conf\iotdb-system.properties" + +if exist "%target_template_file%" ( + del "%target_template_file%" +) + +if exist "%target_properties_file%" ( + del "%target_properties_file%" +) + +mkdir "%target_template_file%\.." + +copy "%source_file%" "%target_template_file%" + +echo # >> "%target_properties_file%" +echo # Licensed to the Apache Software Foundation (ASF) under one >> "%target_properties_file%" +echo # or more contributor license agreements. See the NOTICE file >> "%target_properties_file%" +echo # distributed with this work for additional information >> "%target_properties_file%" +echo # regarding copyright ownership. The ASF licenses this file >> "%target_properties_file%" +echo # to you under the Apache License, Version 2.0 (the >> "%target_properties_file%" +echo # "License"); you may not use this file except in compliance >> "%target_properties_file%" +echo # with the License. You may obtain a copy of the License at >> "%target_properties_file%" +echo # >> "%target_properties_file%" +echo # http://www.apache.org/licenses/LICENSE-2.0 >> "%target_properties_file%" +echo # >> "%target_properties_file%" +echo # Unless required by applicable law or agreed to in writing, >> "%target_properties_file%" +echo # software distributed under the License is distributed on an >> "%target_properties_file%" +echo # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY >> "%target_properties_file%" +echo # KIND, either express or implied. See the License for the >> "%target_properties_file%" +echo # specific language governing permissions and limitations >> "%target_properties_file%" +echo # under the License. >> "%target_properties_file%" +echo # >> "%target_properties_file%" + +for /f "usebackq tokens=*" %%i in ("%target_template_file%") do ( + set "line=%%i" + setlocal enabledelayedexpansion + if not "!line!"=="" if "!line:~0,1!" NEQ "#" ( + echo !line!>>"%target_properties_file%" + ) + endlocal +) \ No newline at end of file diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh new file mode 100644 index 000000000000..f14329b76a6a --- /dev/null +++ b/iotdb-core/node-commons/src/assembly/resources/conf/generate_properties.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +source_file="src/assembly/resources/conf/iotdb-system.properties" +target_template_file="target/conf/iotdb-system.properties.template" +target_properties_file="target/conf/iotdb-system.properties" + +if [ -f "$target_template_file" ]; then + rm "$target_template_file" +fi + +if [ -f "$target_properties_file" ]; then + rm "$target_properties_file" +fi + +mkdir -p "$(dirname "$target_template_file")" + +cp "$source_file" "$target_template_file" + +# Add license header to the target properties file +cat < "$target_properties_file" +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +EOL + +grep -v '^\s*#' "$target_template_file" | grep -v '^\s*$' >> "$target_properties_file" diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties index 958765e523f0..9e32106da527 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties @@ -1658,7 +1658,7 @@ data_replication_factor=1 # The total bytes that all pipe sinks can transfer per second. # When given a value less than or equal to 0, it means no limit. # default value is -1, which means no limit. -# Datatype: int +# Datatype: double # pipe_all_sinks_rate_limit_bytes_per_second=-1 #################### diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java index 3d42596e1fee..f72ea5e528d1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java @@ -562,15 +562,14 @@ private void loadPipeProps(Properties properties) { properties.getProperty( "two_stage_aggregate_sender_end_points_cache_in_ms", String.valueOf(config.getTwoStageAggregateSenderEndPointsCacheInMs())))); + } + private void loadSubscriptionProps(Properties properties) { config.setSubscriptionCacheMemoryUsagePercentage( Float.parseFloat( properties.getProperty( "subscription_cache_memory_usage_percentage", String.valueOf(config.getSubscriptionCacheMemoryUsagePercentage())))); - } - - private void loadSubscriptionProps(Properties properties) { config.setSubscriptionSubtaskExecutorMaxThreadNum( Integer.parseInt( properties.getProperty( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java deleted file mode 100644 index 49b44f0bc02f..000000000000 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigFileAutoUpdateTool.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iotdb.commons.conf; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.net.URL; -import java.nio.file.Files; -import java.util.concurrent.TimeUnit; - -public class ConfigFileAutoUpdateTool { - - private final String lockFileSuffix = ".lock"; - private final long maxTimeMillsToAcquireLock = TimeUnit.SECONDS.toMillis(20); - private final long waitTimeMillsPerCheck = TimeUnit.MILLISECONDS.toMillis(100); - private Logger logger = LoggerFactory.getLogger(ConfigFileAutoUpdateTool.class); - private String license = - "#\n" - + "# Licensed to the Apache Software Foundation (ASF) under one\n" - + "# or more contributor license agreements. See the NOTICE file\n" - + "# distributed with this work for additional information\n" - + "# regarding copyright ownership. The ASF licenses this file\n" - + "# to you under the Apache License, Version 2.0 (the\n" - + "# \"License\"); you may not use this file except in compliance\n" - + "# with the License. You may obtain a copy of the License at\n" - + "#\n" - + "# http://www.apache.org/licenses/LICENSE-2.0\n" - + "#\n" - + "# Unless required by applicable law or agreed to in writing,\n" - + "# software distributed under the License is distributed on an\n" - + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n" - + "# KIND, either express or implied. See the License for the\n" - + "# specific language governing permissions and limitations\n" - + "# under the License."; - - public void checkAndMayUpdate(URL systemUrl, URL configNodeUrl, URL dataNodeUrl, URL commonUrl) - throws IOException, InterruptedException { - if (systemUrl == null || configNodeUrl == null || dataNodeUrl == null || commonUrl == null) { - return; - } - File systemFile = new File(systemUrl.getFile()); - File configNodeFile = new File(configNodeUrl.getFile()); - File dataNodeFile = new File(dataNodeUrl.getFile()); - File commonFile = new File(commonUrl.getFile()); - - if (systemFile.exists()) { - return; - } - boolean canUpdate = (configNodeFile.exists() || dataNodeFile.exists()) && commonFile.exists(); - if (!canUpdate) { - return; - } - - File lockFile = new File(systemFile.getPath() + lockFileSuffix); - acquireTargetFileLock(lockFile); - try { - // other progress updated this file - if (systemFile.exists()) { - return; - } - try (RandomAccessFile raf = new RandomAccessFile(lockFile, "rw")) { - raf.write(license.getBytes()); - String configNodeContent = readConfigLines(configNodeFile); - raf.write(configNodeContent.getBytes()); - String dataNodeContent = readConfigLines(dataNodeFile); - raf.write(dataNodeContent.getBytes()); - String commonContent = readConfigLines(commonFile); - raf.write(commonContent.getBytes()); - } - Files.move(lockFile.toPath(), systemFile.toPath()); - } finally { - releaseFileLock(lockFile); - } - } - - private String readConfigLines(File file) throws IOException { - if (!file.exists()) { - return ""; - } - byte[] bytes = Files.readAllBytes(file.toPath()); - String content = new String(bytes); - return content.replace(license, ""); - } - - private void acquireTargetFileLock(File file) throws IOException, InterruptedException { - long totalWaitTime = 0; - while (totalWaitTime < maxTimeMillsToAcquireLock) { - if (file.createNewFile()) { - return; - } - totalWaitTime += waitTimeMillsPerCheck; - Thread.sleep(waitTimeMillsPerCheck); - } - logger.warn( - "Waiting for {} seconds to acquire configuration file update lock." - + " There may have been an unexpected interruption in the last" - + " configuration file update. Ignore temporary file {}", - totalWaitTime / 1000, - file.getName()); - } - - private void releaseFileLock(File file) throws IOException { - Files.deleteIfExists(file.toPath()); - } -} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java new file mode 100644 index 000000000000..0bd422d46202 --- /dev/null +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/ConfigurationFileUtils.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.commons.conf; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.RandomAccessFile; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +public class ConfigurationFileUtils { + + private static final String lockFileSuffix = ".lock"; + private static final long maxTimeMillsToAcquireLock = TimeUnit.SECONDS.toMillis(20); + private static final long waitTimeMillsPerCheck = TimeUnit.MILLISECONDS.toMillis(100); + private static Logger logger = LoggerFactory.getLogger(ConfigurationFileUtils.class); + private static String license = + "#\n" + + "# Licensed to the Apache Software Foundation (ASF) under one\n" + + "# or more contributor license agreements. See the NOTICE file\n" + + "# distributed with this work for additional information\n" + + "# regarding copyright ownership. The ASF licenses this file\n" + + "# to you under the Apache License, Version 2.0 (the\n" + + "# \"License\"); you may not use this file except in compliance\n" + + "# with the License. You may obtain a copy of the License at\n" + + "#\n" + + "# http://www.apache.org/licenses/LICENSE-2.0\n" + + "#\n" + + "# Unless required by applicable law or agreed to in writing,\n" + + "# software distributed under the License is distributed on an\n" + + "# \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n" + + "# KIND, either express or implied. See the License for the\n" + + "# specific language governing permissions and limitations\n" + + "# under the License."; + + // This is a temporary implementations + private static final Set ignoreConfigKeys = + new HashSet<>( + Arrays.asList( + "cn_internal_address", + "cn_internal_port", + "cn_consensus_port", + "cn_seed_config_node", + "dn_internal_address", + "dn_internal_port", + "dn_mpp_data_exchange_port", + "dn_schema_region_consensus_port", + "dn_data_region_consensus_port", + "dn_seed_config_node", + "dn_session_timeout_threshold", + "config_node_consensus_protocol_class", + "schema_replication_factor", + "data_replication_factor", + "data_region_consensus_protocol_class", + "series_slot_num", + "series_partition_executor_class", + "time_partition_interval", + "schema_engine_mode", + "tag_attribute_flush_interval", + "tag_attribute_total_size", + "timestamp_precision", + "iotdb_server_encrypt_decrypt_provider", + "iotdb_server_encrypt_decrypt_provider_parameter")); + + public static void checkAndMayUpdate( + URL systemUrl, URL configNodeUrl, URL dataNodeUrl, URL commonUrl) + throws IOException, InterruptedException { + if (systemUrl == null || configNodeUrl == null || dataNodeUrl == null || commonUrl == null) { + return; + } + File systemFile = new File(systemUrl.getFile()); + File configNodeFile = new File(configNodeUrl.getFile()); + File dataNodeFile = new File(dataNodeUrl.getFile()); + File commonFile = new File(commonUrl.getFile()); + + if (systemFile.exists()) { + return; + } + boolean canUpdate = (configNodeFile.exists() || dataNodeFile.exists()) && commonFile.exists(); + if (!canUpdate) { + return; + } + + File lockFile = new File(systemFile.getPath() + lockFileSuffix); + acquireTargetFileLock(lockFile); + try { + // other progress updated this file + if (systemFile.exists()) { + return; + } + try (RandomAccessFile raf = new RandomAccessFile(lockFile, "rw")) { + raf.write(license.getBytes()); + String configNodeContent = readConfigLinesWithoutLicense(configNodeFile); + raf.write(configNodeContent.getBytes()); + String dataNodeContent = readConfigLinesWithoutLicense(dataNodeFile); + raf.write(dataNodeContent.getBytes()); + String commonContent = readConfigLinesWithoutLicense(commonFile); + raf.write(commonContent.getBytes()); + } + Files.move(lockFile.toPath(), systemFile.toPath()); + } finally { + releaseFileLock(lockFile); + } + } + + public static String readConfigFileContent(URL url) throws IOException { + if (url == null) { + return ""; + } + File f = new File(url.getFile()); + if (!f.exists()) { + return ""; + } + return readConfigLines(f); + } + + public static String readConfigurationTemplateFile() throws IOException { + StringBuilder content = new StringBuilder(); + try (InputStream inputStream = + ConfigurationFileUtils.class + .getClassLoader() + .getResourceAsStream(CommonConfig.SYSTEM_CONFIG_NAME); + InputStreamReader isr = new InputStreamReader(inputStream); + BufferedReader reader = new BufferedReader(isr)) { + String line; + while ((line = reader.readLine()) != null) { + content.append(line).append(System.lineSeparator()); + } + } catch (IOException e) { + logger.warn("Failed to read configuration template", e); + throw e; + } + return content.toString(); + } + + public static List filterImmutableConfigItems(Map configItems) { + List ignoredConfigItems = new ArrayList<>(); + for (String ignoredKey : ignoreConfigKeys) { + if (configItems.containsKey(ignoredKey)) { + configItems.remove(ignoredKey); + ignoredConfigItems.add(ignoredKey); + } + } + return ignoredConfigItems; + } + + public static void updateConfigurationFile(File file, Properties newConfigItems) + throws IOException, InterruptedException { + // read configuration file + List lines = new ArrayList<>(); + try (BufferedReader reader = new BufferedReader(new FileReader(file))) { + String line = null; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + } + // generate new configuration file content in memory + StringBuilder contentsOfNewConfigurationFile = new StringBuilder(); + for (String currentLine : lines) { + if (currentLine.trim().isEmpty() || currentLine.trim().startsWith("#")) { + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + continue; + } + int equalsIndex = currentLine.indexOf('='); + // replace old config + if (equalsIndex != -1) { + String key = currentLine.substring(0, equalsIndex).trim(); + String value = currentLine.substring(equalsIndex + 1).trim(); + if (!newConfigItems.containsKey(key)) { + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + continue; + } + if (newConfigItems.getProperty(key).equals(value)) { + contentsOfNewConfigurationFile.append(currentLine).append(System.lineSeparator()); + newConfigItems.remove(key); + } else { + contentsOfNewConfigurationFile + .append("#") + .append(currentLine) + .append(System.lineSeparator()); + } + } + } + if (newConfigItems.isEmpty()) { + // No configuration needs to be modified + return; + } + File lockFile = new File(file.getPath() + lockFileSuffix); + acquireTargetFileLock(lockFile); + logger.info("Updating configuration file {}", file.getAbsolutePath()); + try { + try (FileWriter writer = new FileWriter(lockFile)) { + writer.write(contentsOfNewConfigurationFile.toString()); + // add new config items + newConfigItems.store(writer, null); + writer.flush(); + } + Files.move(lockFile.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING); + } finally { + releaseFileLock(lockFile); + } + } + + private static String readConfigLinesWithoutLicense(File file) throws IOException { + if (!file.exists()) { + return ""; + } + byte[] bytes = Files.readAllBytes(file.toPath()); + String content = new String(bytes); + return content.replace(license, ""); + } + + private static String readConfigLines(File file) throws IOException { + if (!file.exists()) { + return ""; + } + byte[] bytes = Files.readAllBytes(file.toPath()); + return new String(bytes); + } + + private static void acquireTargetFileLock(File file) throws IOException, InterruptedException { + long totalWaitTime = 0; + while (totalWaitTime < maxTimeMillsToAcquireLock) { + if (file.createNewFile()) { + return; + } + totalWaitTime += waitTimeMillsPerCheck; + Thread.sleep(waitTimeMillsPerCheck); + } + logger.warn( + "Waiting for {} seconds to acquire configuration file update lock." + + " There may have been an unexpected interruption in the last" + + " configuration file update. Ignore temporary file {}", + totalWaitTime / 1000, + file.getName()); + } + + private static void releaseFileLock(File file) throws IOException { + Files.deleteIfExists(file.toPath()); + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java index 890e46a451a6..494da226e83c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/partition/SeriesPartitionTable.java @@ -33,27 +33,27 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.TreeMap; import java.util.Vector; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; public class SeriesPartitionTable { - private final TreeMap> seriesPartitionMap; + private final ConcurrentSkipListMap> + seriesPartitionMap; public SeriesPartitionTable() { - this.seriesPartitionMap = new TreeMap<>(); + this.seriesPartitionMap = new ConcurrentSkipListMap<>(); } public SeriesPartitionTable(Map> seriesPartitionMap) { - this.seriesPartitionMap = new TreeMap<>(seriesPartitionMap); + this.seriesPartitionMap = new ConcurrentSkipListMap<>(seriesPartitionMap); } public Map> getSeriesPartitionMap() { @@ -61,7 +61,7 @@ public Map> getSeriesPartitionMap() } public void putDataPartition(TTimePartitionSlot timePartitionSlot, TConsensusGroupId groupId) { - seriesPartitionMap.computeIfAbsent(timePartitionSlot, empty -> new ArrayList<>()).add(groupId); + seriesPartitionMap.computeIfAbsent(timePartitionSlot, empty -> new Vector<>()).add(groupId); } /** @@ -75,7 +75,6 @@ public boolean getDataPartition( TTimeSlotList partitionSlotList, SeriesPartitionTable seriesPartitionTable) { AtomicBoolean result = new AtomicBoolean(true); List partitionSlots = partitionSlotList.getTimePartitionSlots(); - if (partitionSlots.isEmpty()) { // Return all DataPartitions in one SeriesPartitionSlot // when the queried TimePartitionSlots are empty @@ -84,7 +83,8 @@ public boolean getDataPartition( boolean isNeedLeftAll = partitionSlotList.isNeedLeftAll(), isNeedRightAll = partitionSlotList.isNeedRightAll(); if (isNeedLeftAll || isNeedRightAll) { - // we need to calculate the leftMargin which contains all the time partition on the unclosed + // we need to calculate the leftMargin which contains all the time partition on the + // unclosed // left side: (-oo, leftMargin) // and the rightMargin which contains all the time partition on the unclosed right side: // (rightMargin, +oo) @@ -106,7 +106,6 @@ public boolean getDataPartition( }) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); } - // Return the DataPartition for each match TimePartitionSlot partitionSlots.forEach( timePartitionSlot -> { @@ -119,7 +118,6 @@ public boolean getDataPartition( } }); } - return result.get(); } @@ -153,7 +151,7 @@ public TConsensusGroupId getPredecessorDataPartition(TTimePartitionSlot timePart * @return the timePartition's corresponding dataRegionIds. return the dataRegions which * timeslotIds are in the time range [startTimeSlotId, endTimeSlotId]. */ - List getRegionId( + public List getRegionId( TTimePartitionSlot startTimeSlotId, TTimePartitionSlot endTimeSlotId) { return seriesPartitionMap.entrySet().stream() .filter( @@ -164,7 +162,7 @@ List getRegionId( .collect(Collectors.toList()); } - List getTimeSlotList( + public List getTimeSlotList( TConsensusGroupId regionId, long startTime, long endTime) { if (regionId.getId() == -1) { return seriesPartitionMap.keySet().stream() @@ -214,14 +212,12 @@ public void createDataPartition( public synchronized List filterUnassignedDataPartitionSlots( List partitionSlots) { List result = new Vector<>(); - partitionSlots.forEach( timePartitionSlot -> { if (!seriesPartitionMap.containsKey(timePartitionSlot)) { result.add(timePartitionSlot); } }); - return result; } @@ -258,13 +254,11 @@ public void deserialize(ByteBuffer buffer) { for (int i = 0; i < timePartitionSlotNum; i++) { TTimePartitionSlot timePartitionSlot = ThriftCommonsSerDeUtils.deserializeTTimePartitionSlot(buffer); - int consensusGroupIdNum = buffer.getInt(); List consensusGroupIds = new Vector<>(); for (int j = 0; j < consensusGroupIdNum; j++) { consensusGroupIds.add(ThriftCommonsSerDeUtils.deserializeTConsensusGroupId(buffer)); } - seriesPartitionMap.put(timePartitionSlot, consensusGroupIds); } } @@ -276,7 +270,6 @@ public void deserialize(InputStream inputStream, TProtocol protocol) for (int i = 0; i < timePartitionSlotNum; i++) { TTimePartitionSlot timePartitionSlot = new TTimePartitionSlot(); timePartitionSlot.read(protocol); - int consensusGroupIdNum = ReadWriteIOUtils.readInt(inputStream); List consensusGroupIds = new Vector<>(); for (int j = 0; j < consensusGroupIdNum; j++) { @@ -284,7 +277,6 @@ public void deserialize(InputStream inputStream, TProtocol protocol) consensusGroupId.read(protocol); consensusGroupIds.add(consensusGroupId); } - seriesPartitionMap.put(timePartitionSlot, consensusGroupIds); } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java index ad8e328e45e0..d81292e34d54 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/protocol/IoTDBConnector.java @@ -117,7 +117,7 @@ public abstract class IoTDBConnector implements PipeConnector { protected PipeReceiverStatusHandler receiverStatusHandler; @Override - public void validate(PipeParameterValidator validator) throws Exception { + public void validate(final PipeParameterValidator validator) throws Exception { final PipeParameters parameters = validator.getParameters(); validator.validate( @@ -232,7 +232,8 @@ public void validate(PipeParameterValidator validator) throws Exception { } @Override - public void customize(PipeParameters parameters, PipeConnectorRuntimeConfiguration configuration) + public void customize( + final PipeParameters parameters, final PipeConnectorRuntimeConfiguration configuration) throws Exception { nodeUrls.clear(); nodeUrls.addAll(parseNodeUrls(parameters)); @@ -276,7 +277,7 @@ public void customize(PipeParameters parameters, PipeConnectorRuntimeConfigurati CONNECTOR_EXCEPTION_OTHERS_RECORD_IGNORED_DATA_DEFAULT_VALUE)); } - protected LinkedHashSet parseNodeUrls(PipeParameters parameters) + protected LinkedHashSet parseNodeUrls(final PipeParameters parameters) throws PipeParameterNotValidException { final LinkedHashSet givenNodeUrls = new LinkedHashSet<>(nodeUrls); @@ -317,15 +318,22 @@ protected LinkedHashSet parseNodeUrls(PipeParameters parameters) givenNodeUrls.addAll( NodeUrlUtils.parseTEndPointUrls( Arrays.asList( - parameters.getStringByKeys(CONNECTOR_IOTDB_NODE_URLS_KEY).split(",")))); + parameters + .getStringByKeys(CONNECTOR_IOTDB_NODE_URLS_KEY) + .replace(" ", "") + .split(",")))); } if (parameters.hasAttribute(SINK_IOTDB_NODE_URLS_KEY)) { givenNodeUrls.addAll( NodeUrlUtils.parseTEndPointUrls( - Arrays.asList(parameters.getStringByKeys(SINK_IOTDB_NODE_URLS_KEY).split(",")))); + Arrays.asList( + parameters + .getStringByKeys(SINK_IOTDB_NODE_URLS_KEY) + .replace(" ", "") + .split(",")))); } - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn(PARSE_URL_ERROR_FORMATTER, e.toString()); throw new PipeParameterNotValidException(PARSE_URL_ERROR_MESSAGE); } @@ -335,8 +343,8 @@ protected LinkedHashSet parseNodeUrls(PipeParameters parameters) return givenNodeUrls; } - private void checkNodeUrls(Set nodeUrls) throws PipeParameterNotValidException { - for (TEndPoint nodeUrl : nodeUrls) { + private void checkNodeUrls(final Set nodeUrls) throws PipeParameterNotValidException { + for (final TEndPoint nodeUrl : nodeUrls) { if (Objects.isNull(nodeUrl.ip) || nodeUrl.ip.isEmpty()) { LOGGER.warn(PARSE_URL_ERROR_FORMATTER, "host cannot be empty"); throw new PipeParameterNotValidException(PARSE_URL_ERROR_MESSAGE); @@ -354,13 +362,13 @@ public void close() { PIPE_END_POINT_RATE_LIMITER_MAP.clear(); } - protected TPipeTransferReq compressIfNeeded(TPipeTransferReq req) throws IOException { + protected TPipeTransferReq compressIfNeeded(final TPipeTransferReq req) throws IOException { return isRpcCompressionEnabled ? PipeTransferCompressedReq.toTPipeTransferReq(req, compressors) : req; } - protected byte[] compressIfNeeded(byte[] reqInBytes) throws IOException { + protected byte[] compressIfNeeded(final byte[] reqInBytes) throws IOException { return isRpcCompressionEnabled ? PipeTransferCompressedReq.toTPipeTransferReqBytes(reqInBytes, compressors) : reqInBytes; diff --git a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift index d81730337c62..49dc50d94c47 100644 --- a/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift +++ b/iotdb-protocol/thrift-commons/src/main/thrift/common.thrift @@ -127,6 +127,11 @@ struct TSetTTLReq { 3: required bool isDataBase } +struct TSetConfigurationReq { + 1: required map configs + 2: required i32 nodeId +} + // for File struct TFile { 1: required string fileName @@ -210,4 +215,15 @@ enum TAggregationType { MAX_BY, MIN_BY, UDAF -} \ No newline at end of file +} + +struct TShowConfigurationTemplateResp { + 1: required TSStatus status + 2: required string content +} + +struct TShowConfigurationResp { + 1: required TSStatus status + 2: required string content +} + diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index cb1932ca6357..b25b94867107 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1329,6 +1329,12 @@ service IConfigNodeRPCService { /** Clear the cache of chunk, chunk metadata and timeseries metadata to release the memory footprint on all DataNodes */ common.TSStatus clearCache() + /** Set configuration on specified node */ + common.TSStatus setConfiguration(common.TSetConfigurationReq req) + + /** Show content of configuration file */ + common.TShowConfigurationResp showConfiguration(1: i32 nodeId) + /** Check and repair unsorted tsfile by compaction */ common.TSStatus startRepairData() diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift index a99a8e599c93..6069199bd64c 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/client.thrift @@ -634,6 +634,10 @@ service IClientRPCService { TSQueryTemplateResp querySchemaTemplate(1:TSQueryTemplateReq req); + common.TShowConfigurationTemplateResp showConfigurationTemplate(); + + common.TShowConfigurationResp showConfiguration(1:i32 nodeId); + common.TSStatus setSchemaTemplate(1:TSSetSchemaTemplateReq req); common.TSStatus unsetSchemaTemplate(1:TSUnsetSchemaTemplateReq req); diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index e7e8af472c0f..637387084097 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -856,6 +856,10 @@ service IDataNodeRPCService { common.TSStatus clearCache() + common.TShowConfigurationResp showConfiguration() + + common.TSStatus setConfiguration(common.TSetConfigurationReq req) + common.TSStatus loadConfiguration() common.TSStatus setSystemStatus(string status)