From f4de0de8e19acf6c7c42906089233a4df34ed217 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Aug 2024 14:55:59 +0800 Subject: [PATCH 001/109] DN Completion --- .../org/apache/iotdb/rpc/TSStatusCode.java | 1 + .../config/TableConfigTaskVisitor.java | 17 +++++ .../executor/ClusterConfigTaskExecutor.java | 64 +++++++++++++++++ .../config/executor/IConfigTaskExecutor.java | 9 +++ .../AlterTableRenameColumnTask.java | 69 +++++++++++++++++++ .../plan/relational/sql/ast/RenameColumn.java | 32 +++++++-- .../relational/sql/parser/AstBuilder.java | 14 +++- .../relational/sql/util/SqlFormatter.java | 8 +++ .../schema/table/AlterTableOperationType.java | 3 +- 9 files changed, 208 insertions(+), 9 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index cb2e8220e904..e01704ce68c9 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -90,6 +90,7 @@ public enum TSStatusCode { TABLE_NOT_EXISTS(550), TABLE_ALREADY_EXISTS(551), COLUMN_ALREADY_EXISTS(552), + COLUMN_NOT_EXISTS(553), ONLY_LOGICAL_VIEW(560), // Storage Engine diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 672ccc5fe2db..44503a525979 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowClusterTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableRenameColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableSetPropertiesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateTableTask; @@ -59,6 +60,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; @@ -211,6 +213,21 @@ protected IConfigTask visitAddColumn(final AddColumn node, final MPPQueryContext node.columnIfNotExists()); } + @Override + protected IConfigTask visitRenameColumn(final RenameColumn node, final MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + final Pair databaseTablePair = splitQualifiedName(node.getTable()); + + return new AlterTableRenameColumnTask( + databaseTablePair.getLeft(), + databaseTablePair.getRight(), + node.getSource().getValue(), + node.getTarget().getValue(), + context.getQueryId().getId(), + node.tableIfExists(), + node.columnIfExists()); + } + @Override protected IConfigTask visitSetProperties( final SetProperties node, final MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 37d18c03e5d0..ec521bd71d73 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -3169,6 +3169,70 @@ public SettableFuture alterTableAddColumn( return future; } + @Override + public SettableFuture alterTableRenameColumn( + final String database, + final String tableName, + final String oldName, + final String newName, + final String queryId, + final boolean tableIfExists, + final boolean columnIfExists) { + final SettableFuture future = SettableFuture.create(); + try (final ConfigNodeClient client = + CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + TSStatus tsStatus; + final TAlterTableReq req = new TAlterTableReq(); + req.setDatabase(database); + req.setTableName(tableName); + req.setQueryId(queryId); + req.setOperationType(AlterTableOperationType.RENAME_COLUMN.getTypeValue()); + + final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(oldName, stream); + ReadWriteIOUtils.write(newName, stream); + } catch (IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + req.setUpdateInfo(stream.toByteArray()); + + do { + try { + tsStatus = client.alterTable(req); + } catch (final TTransportException e) { + if (e.getType() == TTransportException.TIMED_OUT + || e.getCause() instanceof SocketTimeoutException) { + // time out mainly caused by slow execution, wait until + tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } else { + throw e; + } + } + // keep waiting until task ends + } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode()); + + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() + || (TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() == tsStatus.getCode() && tableIfExists) + || (TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode() == tsStatus.getCode() + && columnIfExists)) { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } else { + LOGGER.warn( + "Failed to rename column from {} to {} for table {}.{}, status is {}.", + oldName, + newName, + database, + tableName, + tsStatus); + future.setException(new IoTDBException(tsStatus.getMessage(), tsStatus.getCode())); + } + } catch (final ClientManagerException | TException e) { + future.setException(e); + } + return future; + } + @Override public SettableFuture alterTableSetProperties( final String database, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index 3fb6b769be68..2c407563220f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -293,6 +293,15 @@ SettableFuture alterTableAddColumn( final boolean tableIfExists, final boolean columnIfExists); + SettableFuture alterTableRenameColumn( + final String database, + final String tableName, + final String oldName, + final String newName, + final String queryId, + final boolean tableIfExists, + final boolean columnIfExists); + SettableFuture alterTableSetProperties( final String database, final String tableName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java new file mode 100644 index 000000000000..0f1ff91bf587 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +public class AlterTableRenameColumnTask implements IConfigTask { + + private final String database; + + private final String tableName; + + private final String oldName; + + private final String newName; + + private final String queryId; + + private final boolean tableIfExists; + + private final boolean columnIfExists; + + public AlterTableRenameColumnTask( + String database, + final String tableName, + final String oldName, + final String newName, + final String queryId, + final boolean tableIfExists, + final boolean columnIfExists) { + database = PathUtils.qualifyDatabaseName(database); + this.database = database; + this.tableName = tableName; + this.oldName = oldName; + this.newName = newName; + this.queryId = queryId; + this.tableIfExists = tableIfExists; + this.columnIfExists = columnIfExists; + } + + @Override + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.alterTableRenameColumn( + database, tableName, oldName, newName, queryId, tableIfExists, columnIfExists); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RenameColumn.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RenameColumn.java index be86b506d24b..bbaa5f08911e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RenameColumn.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RenameColumn.java @@ -32,12 +32,22 @@ public final class RenameColumn extends Statement { private final Identifier source; private final Identifier target; + private final boolean tableIfExists; + private final boolean columnIfNotExists; + public RenameColumn( - NodeLocation location, QualifiedName table, Identifier source, Identifier target) { + final NodeLocation location, + final QualifiedName table, + final Identifier source, + final Identifier target, + final boolean tableIfExists, + final boolean columnIfNotExists) { super(requireNonNull(location, "location is null")); this.table = requireNonNull(table, "table is null"); this.source = requireNonNull(source, "source is null"); this.target = requireNonNull(target, "target is null"); + this.tableIfExists = tableIfExists; + this.columnIfNotExists = columnIfNotExists; } public QualifiedName getTable() { @@ -52,8 +62,16 @@ public Identifier getTarget() { return target; } + public boolean tableIfExists() { + return tableIfExists; + } + + public boolean columnIfExists() { + return columnIfNotExists; + } + @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitRenameColumn(this, context); } @@ -63,15 +81,17 @@ public List getChildren() { } @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - RenameColumn that = (RenameColumn) o; - return Objects.equals(table, that.table) + final RenameColumn that = (RenameColumn) o; + return tableIfExists == that.tableIfExists + && columnIfNotExists == that.columnIfNotExists + && Objects.equals(table, that.table) && Objects.equals(source, that.source) && Objects.equals(target, that.target); } @@ -87,6 +107,8 @@ public String toString() { .add("table", table) .add("source", source) .add("target", target) + .add("tableIfExists", tableIfExists) + .add("columnIfExists", columnIfNotExists) .toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index fdb7df45243a..77ff8e5663af 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -322,16 +322,24 @@ public Node visitAddColumn(final RelationalSqlParser.AddColumnContext ctx) { } @Override - public Node visitRenameColumn(RelationalSqlParser.RenameColumnContext ctx) { + public Node visitRenameColumn(final RelationalSqlParser.RenameColumnContext ctx) { return new RenameColumn( getLocation(ctx), getQualifiedName(ctx.tableName), (Identifier) visit(ctx.from), - (Identifier) visit(ctx.to)); + (Identifier) visit(ctx.to), + ctx.EXISTS().stream() + .anyMatch( + node -> + node.getSymbol().getTokenIndex() < ctx.COLUMN().getSymbol().getTokenIndex()), + ctx.EXISTS().stream() + .anyMatch( + node -> + node.getSymbol().getTokenIndex() > ctx.COLUMN().getSymbol().getTokenIndex())); } @Override - public Node visitDropColumn(RelationalSqlParser.DropColumnContext ctx) { + public Node visitDropColumn(final RelationalSqlParser.DropColumnContext ctx) { return new DropColumn( getLocation(ctx), getQualifiedName(ctx.tableName), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java index 42361d09a25b..2d0e5cd91863 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java @@ -638,7 +638,15 @@ private String joinProperties(List properties) { @Override protected Void visitRenameColumn(RenameColumn node, Integer indent) { builder.append("ALTER TABLE "); + if (node.tableIfExists()) { + builder.append("IF EXISTS "); + } + builder.append(formatName(node.getTable())).append(" RENAME COLUMN "); + if (node.columnIfExists()) { + builder.append("IF EXISTS "); + } + builder .append(formatName(node.getSource())) .append(" TO ") diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java index 33e4b3aa36dc..d6dec78cccc2 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java @@ -21,7 +21,8 @@ public enum AlterTableOperationType { ADD_COLUMN((byte) 0), - SET_PROPERTIES((byte) 1); + SET_PROPERTIES((byte) 1), + RENAME_COLUMN((byte) 2); private final byte type; From 87960934ec487c1c0e56075ac14b30e3031c19c5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Aug 2024 15:04:33 +0800 Subject: [PATCH 002/109] Update ClusterConfigTaskExecutor.java --- .../executor/ClusterConfigTaskExecutor.java | 128 +++++++++--------- 1 file changed, 61 insertions(+), 67 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index ec521bd71d73..d971dd03a147 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -3130,28 +3130,15 @@ public SettableFuture alterTableAddColumn( final SettableFuture future = SettableFuture.create(); try (final ConfigNodeClient client = CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TSStatus tsStatus; - final TAlterTableReq req = new TAlterTableReq(); - req.setDatabase(database); - req.setTableName(tableName); - req.setQueryId(queryId); - req.setOperationType(AlterTableOperationType.ADD_COLUMN.getTypeValue()); - req.setUpdateInfo(TsTableColumnSchemaUtil.serialize(columnSchemaList)); - do { - try { - tsStatus = client.alterTable(req); - } catch (final TTransportException e) { - if (e.getType() == TTransportException.TIMED_OUT - || e.getCause() instanceof SocketTimeoutException) { - // time out mainly caused by slow execution, wait until - tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); - } else { - throw e; - } - } - // keep waiting until task ends - } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode()); + final TSStatus tsStatus = + sendAlterReq2ConfigNode( + database, + tableName, + queryId, + AlterTableOperationType.ADD_COLUMN, + TsTableColumnSchemaUtil.serialize(columnSchemaList), + client); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() || (TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() == tsStatus.getCode() && tableIfExists) @@ -3181,36 +3168,23 @@ public SettableFuture alterTableRenameColumn( final SettableFuture future = SettableFuture.create(); try (final ConfigNodeClient client = CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TSStatus tsStatus; - final TAlterTableReq req = new TAlterTableReq(); - req.setDatabase(database); - req.setTableName(tableName); - req.setQueryId(queryId); - req.setOperationType(AlterTableOperationType.RENAME_COLUMN.getTypeValue()); final ByteArrayOutputStream stream = new ByteArrayOutputStream(); try { ReadWriteIOUtils.write(oldName, stream); ReadWriteIOUtils.write(newName, stream); - } catch (IOException ignored) { + } catch (final IOException ignored) { // ByteArrayOutputStream won't throw IOException } - req.setUpdateInfo(stream.toByteArray()); - do { - try { - tsStatus = client.alterTable(req); - } catch (final TTransportException e) { - if (e.getType() == TTransportException.TIMED_OUT - || e.getCause() instanceof SocketTimeoutException) { - // time out mainly caused by slow execution, wait until - tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); - } else { - throw e; - } - } - // keep waiting until task ends - } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode()); + final TSStatus tsStatus = + sendAlterReq2ConfigNode( + database, + tableName, + queryId, + AlterTableOperationType.RENAME_COLUMN, + stream.toByteArray(), + client); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() || (TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() == tsStatus.getCode() && tableIfExists) @@ -3243,35 +3217,22 @@ public SettableFuture alterTableSetProperties( final SettableFuture future = SettableFuture.create(); try (final ConfigNodeClient client = CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - TSStatus tsStatus; - final TAlterTableReq req = new TAlterTableReq(); - req.setDatabase(database); - req.setTableName(tableName); - req.setQueryId(queryId); - req.setOperationType(AlterTableOperationType.SET_PROPERTIES.getTypeValue()); final ByteArrayOutputStream stream = new ByteArrayOutputStream(); try { ReadWriteIOUtils.write(properties, stream); - } catch (IOException ignored) { + } catch (final IOException ignored) { // ByteArrayOutputStream won't throw IOException } - req.setUpdateInfo(stream.toByteArray()); - do { - try { - tsStatus = client.alterTable(req); - } catch (final TTransportException e) { - if (e.getType() == TTransportException.TIMED_OUT - || e.getCause() instanceof SocketTimeoutException) { - // time out mainly caused by slow execution, wait until - tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); - } else { - throw e; - } - } - // keep waiting until task ends - } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode()); + final TSStatus tsStatus = + sendAlterReq2ConfigNode( + database, + tableName, + queryId, + AlterTableOperationType.SET_PROPERTIES, + stream.toByteArray(), + client); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() || (TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() == tsStatus.getCode() && ifExists)) { @@ -3291,14 +3252,47 @@ public SettableFuture alterTableSetProperties( return future; } - public void handlePipeConfigClientExit(String clientId) { + private TSStatus sendAlterReq2ConfigNode( + final String database, + final String tableName, + final String queryId, + final AlterTableOperationType type, + final byte[] updateInfo, + final ConfigNodeClient client) + throws ClientManagerException, TException { + TSStatus tsStatus; + final TAlterTableReq req = new TAlterTableReq(); + req.setDatabase(database); + req.setTableName(tableName); + req.setQueryId(queryId); + req.setOperationType(type.getTypeValue()); + req.setUpdateInfo(updateInfo); + + do { + try { + tsStatus = client.alterTable(req); + } catch (final TTransportException e) { + if (e.getType() == TTransportException.TIMED_OUT + || e.getCause() instanceof SocketTimeoutException) { + // time out mainly caused by slow execution, wait until + tsStatus = RpcUtils.getStatus(TSStatusCode.OVERLAP_WITH_EXISTING_TASK); + } else { + throw e; + } + } + // keep waiting until task ends + } while (TSStatusCode.OVERLAP_WITH_EXISTING_TASK.getStatusCode() == tsStatus.getCode()); + return tsStatus; + } + + public void handlePipeConfigClientExit(final String clientId) { try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { final TSStatus status = configNodeClient.handlePipeConfigClientExit(clientId); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != status.getCode()) { LOGGER.warn("Failed to handlePipeConfigClientExit, status is {}.", status); } - } catch (Exception e) { + } catch (final Exception e) { LOGGER.warn("Failed to handlePipeConfigClientExit.", e); } } From afa7d95393881907ed13d2e22555d7ded4f68c4f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Aug 2024 15:15:17 +0800 Subject: [PATCH 003/109] partial CN --- .../confignode/manager/ConfigManager.java | 2 + .../confignode/manager/ProcedureManager.java | 16 ++++ .../table/RenameTableColumnProcedure.java | 90 +++++++++++++++++++ .../procedure/store/ProcedureFactory.java | 4 + .../procedure/store/ProcedureType.java | 1 + 5 files changed, 113 insertions(+) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 524d3f4cc4f0..8447ba76d9c3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -2440,6 +2440,8 @@ public TSStatus alterTable(final TAlterTableReq req) { return procedureManager.alterTableAddColumn(req); case SET_PROPERTIES: return procedureManager.alterTableSetProperties(req); + case RENAME_COLUMN: + return procedureManager.alterTableRenameColumn(req); default: throw new IllegalArgumentException(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 13c465419f03..90a6ef05b348 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -81,6 +81,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.DropConsumerProcedure; @@ -1322,6 +1323,21 @@ public TSStatus alterTableSetProperties(final TAlterTableReq req) { req.database, req.tableName, req.queryId, ReadWriteIOUtils.readMap(req.updateInfo))); } + public TSStatus alterTableRenameColumn(final TAlterTableReq req) { + return executeWithoutDuplicate( + req.database, + null, + req.tableName, + req.queryId, + ProcedureType.RENAME_TABLE_COLUMN_PROCEDURE, + new RenameTableColumnProcedure( + req.database, + req.tableName, + req.queryId, + ReadWriteIOUtils.readString(req.updateInfo), + ReadWriteIOUtils.readString(req.updateInfo))); + } + private TSStatus executeWithoutDuplicate( final String database, final TsTable table, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java new file mode 100644 index 000000000000..7594d7681e11 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; +import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; +import org.apache.iotdb.confignode.procedure.state.schema.AddTableColumnState; + +import java.io.IOException; + +public class RenameTableColumnProcedure + extends StateMachineProcedure { + + private String database; + + private String tableName; + + private String queryId; + + private String oldName; + private String newName; + private TsTable table; + + public RenameTableColumnProcedure() { + super(); + } + + public RenameTableColumnProcedure( + final String database, + final String tableName, + final String queryId, + final String oldName, + final String newName) { + this.database = database; + this.tableName = tableName; + this.queryId = queryId; + this.oldName = oldName; + this.newName = newName; + } + + @Override + protected Flow executeFromState( + final ConfigNodeProcedureEnv configNodeProcedureEnv, + final AddTableColumnState addTableColumnState) + throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { + return null; + } + + @Override + protected void rollbackState( + final ConfigNodeProcedureEnv configNodeProcedureEnv, + final AddTableColumnState addTableColumnState) + throws IOException, InterruptedException, ProcedureException {} + + @Override + protected AddTableColumnState getState(final int stateId) { + return null; + } + + @Override + protected int getStateId(final AddTableColumnState addTableColumnState) { + return 0; + } + + @Override + protected AddTableColumnState getInitialState() { + return null; + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java index d9ca53e436ed..cbc5034f8141 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java @@ -49,6 +49,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.AlterConsumerGroupProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure; @@ -194,6 +195,9 @@ public Procedure create(ByteBuffer buffer) throws IOException { case SET_TABLE_PROPERTIES_PROCEDURE: procedure = new SetTablePropertiesProcedure(); break; + case RENAME_TABLE_COLUMN_PROCEDURE: + procedure = new RenameTableColumnProcedure(); + break; case CREATE_PIPE_PLUGIN_PROCEDURE: procedure = new CreatePipePluginProcedure(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java index 487ff288895c..02ddb32d4ba3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java @@ -66,6 +66,7 @@ public enum ProcedureType { DROP_TABLE_PROCEDURE((short) 751), ADD_TABLE_COLUMN_PROCEDURE((short) 752), SET_TABLE_PROPERTIES_PROCEDURE((short) 753), + RENAME_TABLE_COLUMN_PROCEDURE((short) 754), // ProcedureId 800-899 is used by IoTDB-Ml From d106726adafa918520d035e6875a03f7a3927c8d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:37:09 +0800 Subject: [PATCH 004/109] Partial refactor --- .../table/AbstractAlterTableProcedure.java | 108 ++++++++++++++++++ .../schema/table/AddTableColumnProcedure.java | 61 +--------- .../table/RenameTableColumnProcedure.java | 17 +-- .../table/SetTablePropertiesProcedure.java | 49 +------- 4 files changed, 115 insertions(+), 120 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java new file mode 100644 index 000000000000..3ab8e14a6595 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.commons.schema.table.TsTable; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public abstract class AbstractAlterTableProcedure + extends StateMachineProcedure { + protected String database; + protected String tableName; + protected String queryId; + + protected TsTable table; + + protected AbstractAlterTableProcedure() {} + + protected AbstractAlterTableProcedure( + final String database, final String tableName, final String queryId) { + this.database = database; + this.tableName = tableName; + this.queryId = queryId; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public String getQueryId() { + return queryId; + } + + @Override + public void serialize(final DataOutputStream stream) throws IOException { + super.serialize(stream); + + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + ReadWriteIOUtils.write(queryId, stream); + + if (Objects.nonNull(table)) { + ReadWriteIOUtils.write(true, stream); + table.serialize(stream); + } else { + ReadWriteIOUtils.write(false, stream); + } + } + + @Override + public void deserialize(final ByteBuffer byteBuffer) { + super.deserialize(byteBuffer); + this.database = ReadWriteIOUtils.readString(byteBuffer); + this.tableName = ReadWriteIOUtils.readString(byteBuffer); + this.queryId = ReadWriteIOUtils.readString(byteBuffer); + + if (ReadWriteIOUtils.readBool(byteBuffer)) { + this.table = TsTable.deserialize(byteBuffer); + } + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final AbstractAlterTableProcedure that = (AbstractAlterTableProcedure) o; + return Objects.equals(database, that.database) + && Objects.equals(tableName, that.tableName) + && Objects.equals(queryId, that.queryId); + } + + @Override + public int hashCode() { + return Objects.hash(database, tableName, queryId); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index 3e77b6dad8bf..d073084df6bd 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -29,14 +29,12 @@ import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; -import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.AddTableColumnState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.Pair; -import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,21 +43,11 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.Map; -import java.util.Objects; -public class AddTableColumnProcedure - extends StateMachineProcedure { +public class AddTableColumnProcedure extends AbstractAlterTableProcedure { private static final Logger LOGGER = LoggerFactory.getLogger(AddTableColumnProcedure.class); - - private String database; - - private String tableName; - - private String queryId; - private List addedColumnList; - private TsTable table; public AddTableColumnProcedure() {} @@ -240,65 +228,18 @@ protected AddTableColumnState getInitialState() { return AddTableColumnState.COLUMN_CHECK; } - public String getDatabase() { - return database; - } - - public String getTableName() { - return tableName; - } - - public String getQueryId() { - return queryId; - } - @Override public void serialize(final DataOutputStream stream) throws IOException { stream.writeShort(ProcedureType.ADD_TABLE_COLUMN_PROCEDURE.getTypeCode()); super.serialize(stream); - ReadWriteIOUtils.write(database, stream); - ReadWriteIOUtils.write(tableName, stream); - ReadWriteIOUtils.write(queryId, stream); - TsTableColumnSchemaUtil.serialize(addedColumnList, stream); - if (Objects.nonNull(table)) { - ReadWriteIOUtils.write(true, stream); - table.serialize(stream); - } else { - ReadWriteIOUtils.write(false, stream); - } } @Override public void deserialize(final ByteBuffer byteBuffer) { super.deserialize(byteBuffer); - this.database = ReadWriteIOUtils.readString(byteBuffer); - this.tableName = ReadWriteIOUtils.readString(byteBuffer); - this.queryId = ReadWriteIOUtils.readString(byteBuffer); this.addedColumnList = TsTableColumnSchemaUtil.deserializeColumnSchemaList(byteBuffer); - if (ReadWriteIOUtils.readBool(byteBuffer)) { - this.table = TsTable.deserialize(byteBuffer); - } - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (!(o instanceof AddTableColumnProcedure)) { - return false; - } - final AddTableColumnProcedure that = (AddTableColumnProcedure) o; - return Objects.equals(database, that.database) - && Objects.equals(tableName, that.tableName) - && Objects.equals(queryId, that.queryId); - } - - @Override - public int hashCode() { - return Objects.hash(database, tableName, queryId); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index 7594d7681e11..97dd9e2a147b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -19,28 +19,17 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table; -import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; -import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; import org.apache.iotdb.confignode.procedure.state.schema.AddTableColumnState; import java.io.IOException; -public class RenameTableColumnProcedure - extends StateMachineProcedure { - - private String database; - - private String tableName; - - private String queryId; - +public class RenameTableColumnProcedure extends AbstractAlterTableProcedure { private String oldName; private String newName; - private TsTable table; public RenameTableColumnProcedure() { super(); @@ -52,9 +41,7 @@ public RenameTableColumnProcedure( final String queryId, final String oldName, final String newName) { - this.database = database; - this.tableName = tableName; - this.queryId = queryId; + super(database, tableName, queryId); this.oldName = oldName; this.newName = newName; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java index 10c6d9760121..c9bccb575c17 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java @@ -27,7 +27,6 @@ import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; -import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.SetTablePropertiesState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; @@ -51,19 +50,12 @@ import static org.apache.iotdb.confignode.procedure.state.schema.SetTablePropertiesState.VALIDATE_TABLE; public class SetTablePropertiesProcedure - extends StateMachineProcedure { + extends AbstractAlterTableProcedure { private static final Logger LOGGER = LoggerFactory.getLogger(SetTablePropertiesProcedure.class); - private String database; - - private String tableName; - - private String queryId; - private Map originalProperties = new HashMap<>(); private Map updatedProperties; - private TsTable table; public SetTablePropertiesProcedure() { super(); @@ -74,9 +66,7 @@ public SetTablePropertiesProcedure( final String tableName, final String queryId, final Map properties) { - this.database = database; - this.tableName = tableName; - this.queryId = queryId; + super(database, tableName, queryId); this.updatedProperties = properties; } @@ -258,49 +248,21 @@ protected SetTablePropertiesState getInitialState() { return VALIDATE_TABLE; } - public String getDatabase() { - return database; - } - - public String getTableName() { - return tableName; - } - - public String getQueryId() { - return queryId; - } - @Override public void serialize(final DataOutputStream stream) throws IOException { stream.writeShort(ProcedureType.SET_TABLE_PROPERTIES_PROCEDURE.getTypeCode()); super.serialize(stream); - ReadWriteIOUtils.write(database, stream); - ReadWriteIOUtils.write(tableName, stream); - ReadWriteIOUtils.write(queryId, stream); - ReadWriteIOUtils.write(originalProperties, stream); ReadWriteIOUtils.write(updatedProperties, stream); - if (Objects.nonNull(table)) { - ReadWriteIOUtils.write(true, stream); - table.serialize(stream); - } else { - ReadWriteIOUtils.write(false, stream); - } } @Override public void deserialize(final ByteBuffer byteBuffer) { super.deserialize(byteBuffer); - this.database = ReadWriteIOUtils.readString(byteBuffer); - this.tableName = ReadWriteIOUtils.readString(byteBuffer); - this.queryId = ReadWriteIOUtils.readString(byteBuffer); this.originalProperties = ReadWriteIOUtils.readMap(byteBuffer); this.updatedProperties = ReadWriteIOUtils.readMap(byteBuffer); - if (ReadWriteIOUtils.readBool(byteBuffer)) { - this.table = TsTable.deserialize(byteBuffer); - } } @Override @@ -312,14 +274,11 @@ public boolean equals(final Object o) { return false; } final SetTablePropertiesProcedure that = (SetTablePropertiesProcedure) o; - return Objects.equals(database, that.database) - && Objects.equals(tableName, that.tableName) - && Objects.equals(updatedProperties, that.updatedProperties) - && Objects.equals(queryId, that.queryId); + return super.equals(o) && Objects.equals(updatedProperties, that.updatedProperties); } @Override public int hashCode() { - return Objects.hash(database, tableName, updatedProperties, queryId); + return Objects.hash(super.hashCode(), updatedProperties); } } From a7dba2b5b853f23944110d66d107cde77c507995 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:58:52 +0800 Subject: [PATCH 005/109] Enrich --- .../table/RenameTableColumnProcedure.java | 35 +++++++++++++++++++ .../table/SetTablePropertiesProcedure.java | 10 ++---- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index 97dd9e2a147b..0a218d78d3ee 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -24,8 +24,14 @@ import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; import org.apache.iotdb.confignode.procedure.state.schema.AddTableColumnState; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; public class RenameTableColumnProcedure extends AbstractAlterTableProcedure { private String oldName; @@ -74,4 +80,33 @@ protected int getStateId(final AddTableColumnState addTableColumnState) { protected AddTableColumnState getInitialState() { return null; } + + @Override + public void serialize(final DataOutputStream stream) throws IOException { + stream.writeShort(ProcedureType.SET_TABLE_PROPERTIES_PROCEDURE.getTypeCode()); + super.serialize(stream); + + ReadWriteIOUtils.write(oldName, stream); + ReadWriteIOUtils.write(newName, stream); + } + + @Override + public void deserialize(final ByteBuffer byteBuffer) { + super.deserialize(byteBuffer); + + this.oldName = ReadWriteIOUtils.readString(byteBuffer); + this.newName = ReadWriteIOUtils.readString(byteBuffer); + } + + @Override + public boolean equals(final Object o) { + return super.equals(o) + && Objects.equals(oldName, ((RenameTableColumnProcedure) o).oldName) + && Objects.equals(newName, ((RenameTableColumnProcedure) o).newName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), oldName, newName); + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java index c9bccb575c17..99698657f1e8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java @@ -267,14 +267,8 @@ public void deserialize(final ByteBuffer byteBuffer) { @Override public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (!(o instanceof SetTablePropertiesProcedure)) { - return false; - } - final SetTablePropertiesProcedure that = (SetTablePropertiesProcedure) o; - return super.equals(o) && Objects.equals(updatedProperties, that.updatedProperties); + return super.equals(o) + && Objects.equals(updatedProperties, ((SetTablePropertiesProcedure) o).updatedProperties); } @Override From 839a00742ff8a8ca17d160c2d4159bd591888395 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Aug 2024 17:04:47 +0800 Subject: [PATCH 006/109] Enrich --- .../confignode/manager/ProcedureManager.java | 22 ++++++------------- .../schema/table/AddTableColumnProcedure.java | 4 +--- .../table/RenameTableColumnProcedure.java | 2 +- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 90a6ef05b348..06041f2a888c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -79,6 +79,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.SetTTLProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.AbstractAlterTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure; @@ -1396,24 +1397,15 @@ private Pair awaitDuplicateTableTask( } break; case ADD_TABLE_COLUMN_PROCEDURE: - final AddTableColumnProcedure addTableColumnProcedure = - (AddTableColumnProcedure) procedure; - if (type == thisType && queryId.equals(addTableColumnProcedure.getQueryId())) { - return new Pair<>(procedure.getProcId(), false); - } - if (database.equals(addTableColumnProcedure.getDatabase()) - && Objects.equals(tableName, addTableColumnProcedure.getTableName())) { - return new Pair<>(-1L, true); - } - break; case SET_TABLE_PROPERTIES_PROCEDURE: - final SetTablePropertiesProcedure setTablePropertiesProcedure = - (SetTablePropertiesProcedure) procedure; - if (type == thisType && queryId.equals(setTablePropertiesProcedure.getQueryId())) { + case RENAME_TABLE_COLUMN_PROCEDURE: + final AbstractAlterTableProcedure alterTableProcedure = + (AbstractAlterTableProcedure) procedure; + if (type == thisType && queryId.equals(alterTableProcedure.getQueryId())) { return new Pair<>(procedure.getProcId(), false); } - if (database.equals(setTablePropertiesProcedure.getDatabase()) - && Objects.equals(tableName, setTablePropertiesProcedure.getTableName())) { + if (database.equals(alterTableProcedure.getDatabase()) + && Objects.equals(tableName, alterTableProcedure.getTableName())) { return new Pair<>(-1L, true); } break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index d073084df6bd..6608b4e51632 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -56,9 +56,7 @@ public AddTableColumnProcedure( final String tableName, final String queryId, final List addedColumnList) { - this.database = database; - this.tableName = tableName; - this.queryId = queryId; + super(database, tableName, queryId); this.addedColumnList = addedColumnList; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index 0a218d78d3ee..4881df892708 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -83,7 +83,7 @@ protected AddTableColumnState getInitialState() { @Override public void serialize(final DataOutputStream stream) throws IOException { - stream.writeShort(ProcedureType.SET_TABLE_PROPERTIES_PROCEDURE.getTypeCode()); + stream.writeShort(ProcedureType.RENAME_TABLE_COLUMN_PROCEDURE.getTypeCode()); super.serialize(stream); ReadWriteIOUtils.write(oldName, stream); From cf5c56a13eed9c22c6bba224480da91acc2ff55f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Aug 2024 17:11:01 +0800 Subject: [PATCH 007/109] Create RenameTableColumnProcedureTest.java --- .../table/RenameTableColumnProcedureTest.java | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedureTest.java diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedureTest.java new file mode 100644 index 000000000000..77f36e68569d --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedureTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class RenameTableColumnProcedureTest { + @Test + public void serializeDeserializeTest() throws IllegalPathException, IOException { + final RenameTableColumnProcedure renameTableColumnProcedure = + new RenameTableColumnProcedure("root.database1", "table1", "0", "oldName", "newName"); + + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); + renameTableColumnProcedure.serialize(dataOutputStream); + + final ByteBuffer byteBuffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); + + Assert.assertEquals( + ProcedureType.RENAME_TABLE_COLUMN_PROCEDURE.getTypeCode(), byteBuffer.getShort()); + + final RenameTableColumnProcedure deserializedProcedure = new RenameTableColumnProcedure(); + deserializedProcedure.deserialize(byteBuffer); + + Assert.assertEquals(renameTableColumnProcedure, deserializedProcedure); + } +} From 8735c709847af1270f77b7e7e3b2aae1908fc2a8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Aug 2024 17:29:56 +0800 Subject: [PATCH 008/109] Refactor --- .../table/AbstractAlterTableProcedure.java | 43 +++++++++++++++++++ .../schema/table/AddTableColumnProcedure.java | 31 ++----------- .../table/SetTablePropertiesProcedure.java | 31 ++----------- 3 files changed, 49 insertions(+), 56 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java index 3ab8e14a6595..64e6948128c0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java @@ -19,19 +19,28 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Map; import java.util.Objects; public abstract class AbstractAlterTableProcedure extends StateMachineProcedure { + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractAlterTableProcedure.class); + protected String database; protected String tableName; protected String queryId; @@ -59,6 +68,40 @@ public String getQueryId() { return queryId; } + protected void commitRelease(final ConfigNodeProcedureEnv env) { + final Map failedResults = + SchemaUtils.commitReleaseTable(database, table.getTableName(), env.getConfigManager()); + if (!failedResults.isEmpty()) { + LOGGER.warn( + "Failed to {} for table {}.{} to DataNode, failure results: {}", + getActionMessage(), + database, + table.getTableName(), + failedResults); + // TODO: Handle commit failure + } + } + + protected void rollbackPreRelease(final ConfigNodeProcedureEnv env) { + final Map failedResults = + SchemaUtils.rollbackPreRelease(database, table.getTableName(), env.getConfigManager()); + + if (!failedResults.isEmpty()) { + // All dataNodes must clear the related schema cache + LOGGER.warn( + "Failed to rollback pre-release {} for table {}.{} info to DataNode, failure results: {}", + getActionMessage(), + database, + table.getTableName(), + failedResults); + setFailure( + new ProcedureException( + new MetadataException("Rollback pre-release " + getActionMessage() + " failed"))); + } + } + + protected abstract String getActionMessage(); + @Override public void serialize(final DataOutputStream stream) throws IOException { super.serialize(stream); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index 6608b4e51632..8d8fc16e1111 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -144,17 +144,9 @@ private void addColumn(final ConfigNodeProcedureEnv env) { } } - private void commitRelease(final ConfigNodeProcedureEnv env) { - final Map failedResults = - SchemaUtils.commitReleaseTable(database, table.getTableName(), env.getConfigManager()); - if (!failedResults.isEmpty()) { - LOGGER.warn( - "Failed to commit column extension info of table {}.{} to DataNode, failure results: {}", - database, - table.getTableName(), - failedResults); - // TODO: Handle commit failure - } + @Override + protected String getActionMessage() { + return "add table column"; } @Override @@ -194,23 +186,6 @@ private void rollbackAddColumn(final ConfigNodeProcedureEnv env) { } } - private void rollbackPreRelease(final ConfigNodeProcedureEnv env) { - final Map failedResults = - SchemaUtils.rollbackPreRelease(database, table.getTableName(), env.getConfigManager()); - - if (!failedResults.isEmpty()) { - // All dataNodes must clear the related schema cache - LOGGER.warn( - "Failed to rollback pre-release column extension info of table {}.{} info to DataNode, failure results: {}", - database, - table.getTableName(), - failedResults); - setFailure( - new ProcedureException( - new MetadataException("Rollback pre-release table column extension info failed"))); - } - } - @Override protected AddTableColumnState getState(final int stateId) { return AddTableColumnState.values()[stateId]; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java index 99698657f1e8..c629fdf1ff48 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java @@ -162,17 +162,9 @@ private void setProperties(final ConfigNodeProcedureEnv env) { } } - private void commitRelease(final ConfigNodeProcedureEnv env) { - final Map failedResults = - SchemaUtils.commitReleaseTable(database, table.getTableName(), env.getConfigManager()); - if (!failedResults.isEmpty()) { - LOGGER.warn( - "Failed to commit properties info of table {}.{} to DataNode, failure results: {}", - database, - table.getTableName(), - failedResults); - // TODO: Handle commit failure - } + @Override + protected String getActionMessage() { + return "set table properties"; } @Override @@ -203,23 +195,6 @@ protected void rollbackState( } } - private void rollbackPreRelease(final ConfigNodeProcedureEnv env) { - final Map failedResults = - SchemaUtils.rollbackPreRelease(database, table.getTableName(), env.getConfigManager()); - - if (!failedResults.isEmpty()) { - // All dataNodes must clear the related schema cache - LOGGER.warn( - "Failed to rollback properties info of table {}.{} info to DataNode, failure results: {}", - database, - table.getTableName(), - failedResults); - setFailure( - new ProcedureException( - new MetadataException("Rollback pre-release table column extension info failed"))); - } - } - private void rollbackSetProperties(final ConfigNodeProcedureEnv env) { if (table == null) { return; From db53ad4acc9ec3635a5b9aadf2e980cf71f38953 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Aug 2024 20:38:59 +0800 Subject: [PATCH 009/109] Refactor --- .../table/AbstractAlterTableProcedure.java | 18 +++++++++++++++ .../schema/table/AddTableColumnProcedure.java | 23 +++---------------- .../table/SetTablePropertiesProcedure.java | 22 +++--------------- 3 files changed, 24 insertions(+), 39 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java index 64e6948128c0..236cb898b9a1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java @@ -68,6 +68,24 @@ public String getQueryId() { return queryId; } + protected void preRelease(final ConfigNodeProcedureEnv env) { + final Map failedResults = + SchemaUtils.preReleaseTable(database, table, env.getConfigManager()); + + if (!failedResults.isEmpty()) { + // All dataNodes must clear the related schema cache + LOGGER.warn( + "Failed to pre-release {} for table {}.{} to DataNode, failure results: {}", + getActionMessage(), + database, + table.getTableName(), + failedResults); + setFailure( + new ProcedureException( + new MetadataException("Pre-release " + getActionMessage() + " failed"))); + } + } + protected void commitRelease(final ConfigNodeProcedureEnv env) { final Map failedResults = SchemaUtils.commitReleaseTable(database, table.getTableName(), env.getConfigManager()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index 8d8fc16e1111..62a39b211731 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -21,7 +21,6 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IoTDBException; -import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; @@ -29,7 +28,6 @@ import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; -import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.AddTableColumnState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.rpc.TSStatusCode; @@ -42,7 +40,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import java.util.Map; public class AddTableColumnProcedure extends AbstractAlterTableProcedure { @@ -112,23 +109,9 @@ private void columnCheck(final ConfigNodeProcedureEnv env) { setNextState(AddTableColumnState.PRE_RELEASE); } - private void preRelease(final ConfigNodeProcedureEnv env) { - final Map failedResults = - SchemaUtils.preReleaseTable(database, table, env.getConfigManager()); - - if (!failedResults.isEmpty()) { - // All dataNodes must clear the related schema cache - LOGGER.warn( - "Failed to pre-release column extension info of table {}.{} to DataNode, failure results: {}", - database, - table.getTableName(), - failedResults); - setFailure( - new ProcedureException( - new MetadataException("Pre-release table column extension info failed"))); - return; - } - + @Override + protected void preRelease(final ConfigNodeProcedureEnv env) { + super.preRelease(env); setNextState(AddTableColumnState.ADD_COLUMN); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java index c629fdf1ff48..37ac5a71a884 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java @@ -21,13 +21,11 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IoTDBException; -import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; -import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.SetTablePropertiesState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.rpc.TSStatusCode; @@ -130,23 +128,9 @@ private void validateTable(final ConfigNodeProcedureEnv env) { setNextState(PRE_RELEASE); } - private void preRelease(final ConfigNodeProcedureEnv env) { - final Map failedResults = - SchemaUtils.preReleaseTable(database, table, env.getConfigManager()); - - if (!failedResults.isEmpty()) { - // All dataNodes must clear the related schema cache - LOGGER.warn( - "Failed to pre-release properties info of table {}.{} to DataNode, failure results: {}", - database, - table.getTableName(), - failedResults); - setFailure( - new ProcedureException( - new MetadataException("Pre-release table properties info failed"))); - return; - } - + @Override + protected void preRelease(final ConfigNodeProcedureEnv env) { + super.preRelease(env); setNextState(SET_PROPERTIES); } From 9b7d312123a119329be65f95a8b050b5cc9cc487 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Aug 2024 20:53:08 +0800 Subject: [PATCH 010/109] Drop on dn --- .../executor/ClusterConfigTaskExecutor.java | 48 ++++++++++++++ .../config/executor/IConfigTaskExecutor.java | 8 +++ .../relational/AlterTableDropColumnTask.java | 65 +++++++++++++++++++ .../plan/relational/sql/ast/DropColumn.java | 34 ++++++++-- .../relational/sql/parser/AstBuilder.java | 10 ++- .../relational/sql/util/SqlFormatter.java | 10 ++- 6 files changed, 168 insertions(+), 7 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index d971dd03a147..b54c4183cc62 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -3207,6 +3207,54 @@ public SettableFuture alterTableRenameColumn( return future; } + @Override + public SettableFuture alterTableDropColumn( + final String database, + final String tableName, + final String columnName, + final String queryId, + final boolean tableIfExists, + final boolean columnIfExists) { + final SettableFuture future = SettableFuture.create(); + try (final ConfigNodeClient client = + CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + + final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(columnName, stream); + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + + final TSStatus tsStatus = + sendAlterReq2ConfigNode( + database, + tableName, + queryId, + AlterTableOperationType.RENAME_COLUMN, + stream.toByteArray(), + client); + + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() + || (TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() == tsStatus.getCode() && tableIfExists) + || (TSStatusCode.COLUMN_NOT_EXISTS.getStatusCode() == tsStatus.getCode() + && columnIfExists)) { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } else { + LOGGER.warn( + "Failed to drop column {} for table {}.{}, status is {}.", + columnName, + database, + tableName, + tsStatus); + future.setException(new IoTDBException(tsStatus.getMessage(), tsStatus.getCode())); + } + } catch (final ClientManagerException | TException e) { + future.setException(e); + } + return future; + } + @Override public SettableFuture alterTableSetProperties( final String database, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index 2c407563220f..a530759adafd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -302,6 +302,14 @@ SettableFuture alterTableRenameColumn( final boolean tableIfExists, final boolean columnIfExists); + SettableFuture alterTableDropColumn( + final String database, + final String tableName, + final String columnName, + final String queryId, + final boolean tableIfExists, + final boolean columnIfExists); + SettableFuture alterTableSetProperties( final String database, final String tableName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java new file mode 100644 index 000000000000..8eef88a39b27 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +public class AlterTableDropColumnTask implements IConfigTask { + + private final String database; + + private final String tableName; + + private final String columnName; + + private final String queryId; + + private final boolean tableIfExists; + + private final boolean columnIfExists; + + public AlterTableDropColumnTask( + String database, + final String tableName, + final String columnName, + final String queryId, + final boolean tableIfExists, + final boolean columnIfExists) { + database = PathUtils.qualifyDatabaseName(database); + this.database = database; + this.tableName = tableName; + this.columnName = columnName; + this.queryId = queryId; + this.tableIfExists = tableIfExists; + this.columnIfExists = columnIfExists; + } + + @Override + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.alterTableDropColumn( + database, tableName, columnName, queryId, tableIfExists, columnIfExists); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java index 28c5a91258a4..049927b37593 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java @@ -32,16 +32,32 @@ public class DropColumn extends Statement { private final QualifiedName table; private final Identifier field; - public DropColumn(QualifiedName table, Identifier field) { + private final boolean tableIfExists; + private final boolean columnIfNotExists; + + public DropColumn( + final QualifiedName table, + final Identifier field, + final boolean tableIfExists, + final boolean columnIfExists) { super(null); this.table = requireNonNull(table, "table is null"); this.field = requireNonNull(field, "field is null"); + this.tableIfExists = tableIfExists; + this.columnIfNotExists = columnIfExists; } - public DropColumn(NodeLocation location, QualifiedName table, Identifier field) { + public DropColumn( + final NodeLocation location, + final QualifiedName table, + final Identifier field, + final boolean tableIfExists, + final boolean columnIfExists) { super(requireNonNull(location, "location is null")); this.table = requireNonNull(table, "table is null"); this.field = requireNonNull(field, "field is null"); + this.tableIfExists = tableIfExists; + this.columnIfNotExists = columnIfExists; } public QualifiedName getTable() { @@ -52,8 +68,16 @@ public Identifier getField() { return field; } + public boolean tableIfExists() { + return tableIfExists; + } + + public boolean columnIfNotExists() { + return columnIfNotExists; + } + @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitDropColumn(this, context); } @@ -63,14 +87,14 @@ public List getChildren() { } @Override - public boolean equals(Object o) { + public boolean equals(final Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } - DropColumn that = (DropColumn) o; + final DropColumn that = (DropColumn) o; return Objects.equals(table, that.table) && Objects.equals(field, that.field); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 77ff8e5663af..d8bd43c443e8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -343,7 +343,15 @@ public Node visitDropColumn(final RelationalSqlParser.DropColumnContext ctx) { return new DropColumn( getLocation(ctx), getQualifiedName(ctx.tableName), - lowerIdentifier((Identifier) visit(ctx.column))); + lowerIdentifier((Identifier) visit(ctx.column)), + ctx.EXISTS().stream() + .anyMatch( + node -> + node.getSymbol().getTokenIndex() < ctx.COLUMN().getSymbol().getTokenIndex()), + ctx.EXISTS().stream() + .anyMatch( + node -> + node.getSymbol().getTokenIndex() > ctx.COLUMN().getSymbol().getTokenIndex())); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java index 2d0e5cd91863..cbd358c2cadb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java @@ -656,9 +656,17 @@ protected Void visitRenameColumn(RenameColumn node, Integer indent) { } @Override - protected Void visitDropColumn(DropColumn node, Integer indent) { + protected Void visitDropColumn(final DropColumn node, final Integer indent) { builder.append("ALTER TABLE "); + if (node.tableIfExists()) { + builder.append("IF EXISTS "); + } + builder.append(formatName(node.getTable())).append(" DROP COLUMN "); + if (node.columnIfNotExists()) { + builder.append("IF NOT EXISTS "); + } + builder.append(formatName(node.getField())); return null; From 6c065d8566e40f7da99024ffe67663000a35dfe2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 22 Aug 2024 20:54:35 +0800 Subject: [PATCH 011/109] fix --- .../execution/config/executor/ClusterConfigTaskExecutor.java | 2 +- .../iotdb/commons/schema/table/AlterTableOperationType.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index b54c4183cc62..185e66300f38 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -3231,7 +3231,7 @@ public SettableFuture alterTableDropColumn( database, tableName, queryId, - AlterTableOperationType.RENAME_COLUMN, + AlterTableOperationType.DROP_COLUMN, stream.toByteArray(), client); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java index d6dec78cccc2..1b2345a4b5a8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java @@ -22,7 +22,8 @@ public enum AlterTableOperationType { ADD_COLUMN((byte) 0), SET_PROPERTIES((byte) 1), - RENAME_COLUMN((byte) 2); + RENAME_COLUMN((byte) 2), + DROP_COLUMN((byte) 3); private final byte type; From 0aae6718a713c8c33844c2916aa5daa605a2d99c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 23 Aug 2024 09:43:37 +0800 Subject: [PATCH 012/109] Refactor --- .../relational/AbstractAlterTableTask.java | 43 +++++++++++++++++++ .../relational/AlterTableAddColumnTask.java | 22 ++-------- .../relational/AlterTableDropColumnTask.java | 21 ++------- .../AlterTableRenameColumnTask.java | 20 ++------- .../AlterTableSetPropertiesTask.java | 22 ++-------- 5 files changed, 56 insertions(+), 72 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractAlterTableTask.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractAlterTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractAlterTableTask.java new file mode 100644 index 000000000000..6fc1ef752cb3 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractAlterTableTask.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.commons.utils.PathUtils; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; + +abstract class AbstractAlterTableTask implements IConfigTask { + + protected final String database; + + protected final String tableName; + + protected final String queryId; + + protected final boolean tableIfExists; + + protected AbstractAlterTableTask( + String database, final String tableName, final String queryId, final boolean tableIfExists) { + database = PathUtils.qualifyDatabaseName(database); + this.database = database; + this.tableName = tableName; + this.queryId = queryId; + this.tableIfExists = tableIfExists; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java index a6e2c7ab21a3..20269554872b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java @@ -20,42 +20,26 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; -import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; -import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; import com.google.common.util.concurrent.ListenableFuture; import java.util.List; -public class AlterTableAddColumnTask implements IConfigTask { - - private final String database; - - private final String tableName; - +public class AlterTableAddColumnTask extends AbstractAlterTableTask { private final List columnList; - - private final String queryId; - - private final boolean tableIfExists; - private final boolean columnIfExists; public AlterTableAddColumnTask( - String database, + final String database, final String tableName, final List columnList, final String queryId, final boolean tableIfExists, final boolean columnIfExists) { - database = PathUtils.qualifyDatabaseName(database); - this.database = database; - this.tableName = tableName; + super(database, tableName, queryId, tableIfExists); this.columnList = columnList; - this.queryId = queryId; - this.tableIfExists = tableIfExists; this.columnIfExists = columnIfExists; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java index 8eef88a39b27..dc136570f53e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java @@ -19,40 +19,25 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; -import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; -import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; import com.google.common.util.concurrent.ListenableFuture; -public class AlterTableDropColumnTask implements IConfigTask { - - private final String database; - - private final String tableName; +public class AlterTableDropColumnTask extends AbstractAlterTableTask { private final String columnName; - - private final String queryId; - - private final boolean tableIfExists; - private final boolean columnIfExists; public AlterTableDropColumnTask( - String database, + final String database, final String tableName, final String columnName, final String queryId, final boolean tableIfExists, final boolean columnIfExists) { - database = PathUtils.qualifyDatabaseName(database); - this.database = database; - this.tableName = tableName; + super(database, tableName, queryId, tableIfExists); this.columnName = columnName; - this.queryId = queryId; - this.tableIfExists = tableIfExists; this.columnIfExists = columnIfExists; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java index 0f1ff91bf587..9caee19c96dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java @@ -19,44 +19,30 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; -import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; -import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; import com.google.common.util.concurrent.ListenableFuture; -public class AlterTableRenameColumnTask implements IConfigTask { - - private final String database; - - private final String tableName; +public class AlterTableRenameColumnTask extends AbstractAlterTableTask { private final String oldName; private final String newName; - private final String queryId; - - private final boolean tableIfExists; - private final boolean columnIfExists; public AlterTableRenameColumnTask( - String database, + final String database, final String tableName, final String oldName, final String newName, final String queryId, final boolean tableIfExists, final boolean columnIfExists) { - database = PathUtils.qualifyDatabaseName(database); - this.database = database; - this.tableName = tableName; + super(database, tableName, queryId, tableIfExists); this.oldName = oldName; this.newName = newName; - this.queryId = queryId; - this.tableIfExists = tableIfExists; this.columnIfExists = columnIfExists; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableSetPropertiesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableSetPropertiesTask.java index 4d650c3a53c7..cf927eb625db 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableSetPropertiesTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableSetPropertiesTask.java @@ -19,45 +19,31 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; -import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; -import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; import com.google.common.util.concurrent.ListenableFuture; import java.util.Map; -public class AlterTableSetPropertiesTask implements IConfigTask { - - private final String database; - - private final String tableName; +public class AlterTableSetPropertiesTask extends AbstractAlterTableTask { private final Map properties; - private final String queryId; - - private final boolean ifExists; - public AlterTableSetPropertiesTask( - String database, + final String database, final String tableName, final Map properties, final String queryId, final boolean ifExists) { - database = PathUtils.qualifyDatabaseName(database); - this.database = database; - this.tableName = tableName; + super(database, tableName, queryId, ifExists); this.properties = properties; - this.queryId = queryId; - this.ifExists = ifExists; } @Override public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.alterTableSetProperties( - database, tableName, properties, queryId, ifExists); + database, tableName, properties, queryId, tableIfExists); } } From 64f8becfcff7d8d809c7909ba8d082d91aa5bf94 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 23 Aug 2024 09:49:37 +0800 Subject: [PATCH 013/109] Completion --- .../config/TableConfigTaskVisitor.java | 16 +++++++++++++ .../plan/relational/sql/ast/AddColumn.java | 2 +- .../plan/relational/sql/ast/DropColumn.java | 24 ++++++++++++------- .../relational/sql/util/SqlFormatter.java | 2 +- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 44503a525979..610305af0e24 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowClusterTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.ShowRegionTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableDropColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableRenameColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableSetPropertiesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask; @@ -51,6 +52,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.CurrentDatabase; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DataType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DescribeTable; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; @@ -228,6 +230,20 @@ protected IConfigTask visitRenameColumn(final RenameColumn node, final MPPQueryC node.columnIfExists()); } + @Override + protected IConfigTask visitDropColumn(final DropColumn node, final MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + final Pair databaseTablePair = splitQualifiedName(node.getTable()); + + return new AlterTableDropColumnTask( + databaseTablePair.getLeft(), + databaseTablePair.getRight(), + node.getField().getValue(), + context.getQueryId().getId(), + node.tableIfExists(), + node.columnIfExists()); + } + @Override protected IConfigTask visitSetProperties( final SetProperties node, final MPPQueryContext context) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AddColumn.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AddColumn.java index a9cae506bc0d..bfff28fdad2a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AddColumn.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AddColumn.java @@ -113,7 +113,7 @@ public String toString() { .add("tableName", tableName) .add("column", column) .add("tableIfExists", tableIfExists) - .add("columnIfExists", columnIfNotExists) + .add("columnIfNotExists", columnIfNotExists) .toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java index 049927b37593..bd776614ab7a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropColumn.java @@ -33,7 +33,7 @@ public class DropColumn extends Statement { private final Identifier field; private final boolean tableIfExists; - private final boolean columnIfNotExists; + private final boolean columnIfExists; public DropColumn( final QualifiedName table, @@ -44,7 +44,7 @@ public DropColumn( this.table = requireNonNull(table, "table is null"); this.field = requireNonNull(field, "field is null"); this.tableIfExists = tableIfExists; - this.columnIfNotExists = columnIfExists; + this.columnIfExists = columnIfExists; } public DropColumn( @@ -57,7 +57,7 @@ public DropColumn( this.table = requireNonNull(table, "table is null"); this.field = requireNonNull(field, "field is null"); this.tableIfExists = tableIfExists; - this.columnIfNotExists = columnIfExists; + this.columnIfExists = columnIfExists; } public QualifiedName getTable() { @@ -72,8 +72,8 @@ public boolean tableIfExists() { return tableIfExists; } - public boolean columnIfNotExists() { - return columnIfNotExists; + public boolean columnIfExists() { + return columnIfExists; } @Override @@ -95,16 +95,24 @@ public boolean equals(final Object o) { return false; } final DropColumn that = (DropColumn) o; - return Objects.equals(table, that.table) && Objects.equals(field, that.field); + return tableIfExists == that.tableIfExists + && columnIfExists == that.columnIfExists + && Objects.equals(table, that.table) + && Objects.equals(field, that.field); } @Override public int hashCode() { - return Objects.hash(table, field); + return Objects.hash(table, field, tableIfExists, columnIfExists); } @Override public String toString() { - return toStringHelper(this).add("table", table).add("field", field).toString(); + return toStringHelper(this) + .add("table", table) + .add("field", field) + .add("tableIfExists", tableIfExists) + .add("columnIfExists", columnIfExists) + .toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java index cbd358c2cadb..778e03fe422a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java @@ -663,7 +663,7 @@ protected Void visitDropColumn(final DropColumn node, final Integer indent) { } builder.append(formatName(node.getTable())).append(" DROP COLUMN "); - if (node.columnIfNotExists()) { + if (node.columnIfExists()) { builder.append("IF NOT EXISTS "); } From ec423cc7c3d1bcc4a241e6da904f1a55a850201d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 23 Aug 2024 10:03:38 +0800 Subject: [PATCH 014/109] Refactor --- .../confignode/manager/ConfigManager.java | 2 + .../confignode/manager/ProcedureManager.java | 13 ++ .../table/DropTableColumnProcedure.java | 111 ++++++++++++++++++ .../table/RenameTableColumnProcedure.java | 5 + .../state/schema/DropTableColumnState.java | 24 ++++ .../state/schema/RenameTableColumnState.java | 24 ++++ .../procedure/store/ProcedureType.java | 1 + 7 files changed, 180 insertions(+) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/RenameTableColumnState.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 8447ba76d9c3..67635a2e15c6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -2442,6 +2442,8 @@ public TSStatus alterTable(final TAlterTableReq req) { return procedureManager.alterTableSetProperties(req); case RENAME_COLUMN: return procedureManager.alterTableRenameColumn(req); + case DROP_COLUMN: + return procedureManager.alterTableDropColumn(req); default: throw new IllegalArgumentException(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 06041f2a888c..ca9ba1b2587d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -82,6 +82,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.table.AbstractAlterTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure; @@ -1339,6 +1340,17 @@ public TSStatus alterTableRenameColumn(final TAlterTableReq req) { ReadWriteIOUtils.readString(req.updateInfo))); } + public TSStatus alterTableDropColumn(final TAlterTableReq req) { + return executeWithoutDuplicate( + req.database, + null, + req.tableName, + req.queryId, + ProcedureType.DROP_TABLE_COLUMN_PROCEDURE, + new DropTableColumnProcedure( + req.database, req.tableName, req.queryId, ReadWriteIOUtils.readString(req.updateInfo))); + } + private TSStatus executeWithoutDuplicate( final String database, final TsTable table, @@ -1399,6 +1411,7 @@ private Pair awaitDuplicateTableTask( case ADD_TABLE_COLUMN_PROCEDURE: case SET_TABLE_PROPERTIES_PROCEDURE: case RENAME_TABLE_COLUMN_PROCEDURE: + case DROP_TABLE_COLUMN_PROCEDURE: final AbstractAlterTableProcedure alterTableProcedure = (AbstractAlterTableProcedure) procedure; if (type == thisType && queryId.equals(alterTableProcedure.getQueryId())) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java new file mode 100644 index 000000000000..4b2255c415ee --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; +import org.apache.iotdb.confignode.procedure.state.schema.DropTableColumnState; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +public class DropTableColumnProcedure extends AbstractAlterTableProcedure { + + private String columnName; + + public DropTableColumnProcedure() { + super(); + } + + public DropTableColumnProcedure( + final String database, + final String tableName, + final String queryId, + final String columnName) { + super(database, tableName, queryId); + this.columnName = columnName; + } + + @Override + protected String getActionMessage() { + return null; + } + + @Override + protected Flow executeFromState( + final ConfigNodeProcedureEnv configNodeProcedureEnv, + final DropTableColumnState dropTableColumnState) + throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { + return null; + } + + @Override + protected void rollbackState( + final ConfigNodeProcedureEnv configNodeProcedureEnv, + final DropTableColumnState dropTableColumnState) + throws IOException, InterruptedException, ProcedureException {} + + @Override + protected DropTableColumnState getState(final int stateId) { + return null; + } + + @Override + protected int getStateId(final DropTableColumnState dropTableColumnState) { + return 0; + } + + @Override + protected DropTableColumnState getInitialState() { + return null; + } + + @Override + public void serialize(final DataOutputStream stream) throws IOException { + stream.writeShort(ProcedureType.DROP_TABLE_COLUMN_PROCEDURE.getTypeCode()); + super.serialize(stream); + + ReadWriteIOUtils.write(columnName, stream); + } + + @Override + public void deserialize(final ByteBuffer byteBuffer) { + super.deserialize(byteBuffer); + + this.columnName = ReadWriteIOUtils.readString(byteBuffer); + } + + @Override + public boolean equals(final Object o) { + return super.equals(o) && Objects.equals(columnName, ((DropTableColumnProcedure) o).columnName); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), columnName); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index 4881df892708..163b7e17530f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -60,6 +60,11 @@ protected Flow executeFromState( return null; } + @Override + protected String getActionMessage() { + return "rename table column"; + } + @Override protected void rollbackState( final ConfigNodeProcedureEnv configNodeProcedureEnv, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java new file mode 100644 index 000000000000..1d0f88c0e9c2 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.state.schema; + +public class DropTableColumnState { + // TODO +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/RenameTableColumnState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/RenameTableColumnState.java new file mode 100644 index 000000000000..58f34283199a --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/RenameTableColumnState.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.state.schema; + +public class RenameTableColumnState { + // TODO +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java index 02ddb32d4ba3..d019898d7547 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureType.java @@ -67,6 +67,7 @@ public enum ProcedureType { ADD_TABLE_COLUMN_PROCEDURE((short) 752), SET_TABLE_PROPERTIES_PROCEDURE((short) 753), RENAME_TABLE_COLUMN_PROCEDURE((short) 754), + DROP_TABLE_COLUMN_PROCEDURE((short) 755), // ProcedureId 800-899 is used by IoTDB-Ml From 8b7bb7f57413b5deaa16cc6c35f7aa65938fa42a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 23 Aug 2024 10:10:16 +0800 Subject: [PATCH 015/109] Add test --- .../table/RenameTableColumnProcedure.java | 25 ++++----- .../procedure/store/ProcedureFactory.java | 4 ++ .../table/DropTableColumnProcedureTest.java | 53 +++++++++++++++++++ 3 files changed, 70 insertions(+), 12 deletions(-) create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedureTest.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index 163b7e17530f..18e0c1d5dc31 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -23,7 +23,7 @@ import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; -import org.apache.iotdb.confignode.procedure.state.schema.AddTableColumnState; +import org.apache.iotdb.confignode.procedure.state.schema.RenameTableColumnState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -33,7 +33,8 @@ import java.nio.ByteBuffer; import java.util.Objects; -public class RenameTableColumnProcedure extends AbstractAlterTableProcedure { +public class RenameTableColumnProcedure + extends AbstractAlterTableProcedure { private String oldName; private String newName; @@ -55,37 +56,37 @@ public RenameTableColumnProcedure( @Override protected Flow executeFromState( final ConfigNodeProcedureEnv configNodeProcedureEnv, - final AddTableColumnState addTableColumnState) + final RenameTableColumnState renameTableColumnState) throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { return null; } - @Override - protected String getActionMessage() { - return "rename table column"; - } - @Override protected void rollbackState( final ConfigNodeProcedureEnv configNodeProcedureEnv, - final AddTableColumnState addTableColumnState) + final RenameTableColumnState renameTableColumnState) throws IOException, InterruptedException, ProcedureException {} @Override - protected AddTableColumnState getState(final int stateId) { + protected RenameTableColumnState getState(final int stateId) { return null; } @Override - protected int getStateId(final AddTableColumnState addTableColumnState) { + protected int getStateId(final RenameTableColumnState renameTableColumnState) { return 0; } @Override - protected AddTableColumnState getInitialState() { + protected RenameTableColumnState getInitialState() { return null; } + @Override + protected String getActionMessage() { + return "rename table column"; + } + @Override public void serialize(final DataOutputStream stream) throws IOException { stream.writeShort(ProcedureType.RENAME_TABLE_COLUMN_PROCEDURE.getTypeCode()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java index cbc5034f8141..97855bf2f827 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java @@ -49,6 +49,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.AlterConsumerGroupProcedure; @@ -198,6 +199,9 @@ public Procedure create(ByteBuffer buffer) throws IOException { case RENAME_TABLE_COLUMN_PROCEDURE: procedure = new RenameTableColumnProcedure(); break; + case DROP_TABLE_COLUMN_PROCEDURE: + procedure = new DropTableColumnProcedure(); + break; case CREATE_PIPE_PLUGIN_PROCEDURE: procedure = new CreatePipePluginProcedure(); break; diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedureTest.java new file mode 100644 index 000000000000..fa1f268233a6 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedureTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class DropTableColumnProcedureTest { + @Test + public void serializeDeserializeTest() throws IllegalPathException, IOException { + final DropTableColumnProcedure dropTableColumnProcedure = + new DropTableColumnProcedure("root.database1", "table1", "0", "columnName"); + + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); + dropTableColumnProcedure.serialize(dataOutputStream); + + final ByteBuffer byteBuffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); + + Assert.assertEquals( + ProcedureType.DROP_TABLE_COLUMN_PROCEDURE.getTypeCode(), byteBuffer.getShort()); + + final DropTableColumnProcedure deserializedProcedure = new DropTableColumnProcedure(); + deserializedProcedure.deserialize(byteBuffer); + + Assert.assertEquals(dropTableColumnProcedure, deserializedProcedure); + } +} From 3471f44e55166227ba8c8005d3e5516d0ab5f6bf Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 23 Aug 2024 10:27:41 +0800 Subject: [PATCH 016/109] Add rename execution --- .../config/TableConfigTaskVisitor.java | 15 ++++++ .../executor/ClusterConfigTaskExecutor.java | 46 ++++++++++++++++++ .../config/executor/IConfigTaskExecutor.java | 7 +++ .../relational/AlterTableRenameTableTask.java | 47 +++++++++++++++++++ .../plan/relational/sql/ast/RenameTable.java | 39 +++++++++++---- .../relational/sql/parser/AstBuilder.java | 5 +- .../relational/sql/util/SqlFormatter.java | 6 ++- .../schema/table/AlterTableOperationType.java | 9 +++- 8 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameTableTask.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 610305af0e24..3daef1fbefbb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableAddColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableDropColumnTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableRenameColumnTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableRenameTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.AlterTableSetPropertiesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateTableTask; @@ -63,6 +64,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetConfiguration; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowCluster; @@ -196,6 +198,19 @@ protected IConfigTask visitCreateTable(final CreateTable node, final MPPQueryCon return new CreateTableTask(table, databaseTablePair.getLeft(), node.isIfNotExists()); } + @Override + protected IConfigTask visitRenameTable(final RenameTable node, final MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + final Pair databaseTablePair = splitQualifiedName(node.getSource()); + + return new AlterTableRenameTableTask( + databaseTablePair.getLeft(), + databaseTablePair.getRight(), + node.getTarget().getValue(), + context.getQueryId().getId(), + node.tableIfExists()); + } + @Override protected IConfigTask visitAddColumn(final AddColumn node, final MPPQueryContext context) { context.setQueryType(QueryType.WRITE); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 185e66300f38..e4262fb5b8cd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -3119,6 +3119,52 @@ public SettableFuture showTables(final String database) { return future; } + @Override + public SettableFuture alterTableRenameTable( + final String database, + final String sourceName, + final String targetName, + final String queryId, + final boolean tableIfExists) { + final SettableFuture future = SettableFuture.create(); + try (final ConfigNodeClient client = + CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + + final ByteArrayOutputStream stream = new ByteArrayOutputStream(); + try { + ReadWriteIOUtils.write(targetName, stream); + } catch (final IOException ignored) { + // ByteArrayOutputStream won't throw IOException + } + + final TSStatus tsStatus = + sendAlterReq2ConfigNode( + database, + sourceName, + queryId, + AlterTableOperationType.RENAME_TABLE, + stream.toByteArray(), + client); + + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() + || (TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() == tsStatus.getCode() + && tableIfExists)) { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } else { + LOGGER.warn( + "Failed to rename table to {} from table {}.{}, status is {}.", + targetName, + database, + sourceName, + tsStatus); + future.setException(new IoTDBException(tsStatus.getMessage(), tsStatus.getCode())); + } + } catch (final ClientManagerException | TException e) { + future.setException(e); + } + return future; + } + @Override public SettableFuture alterTableAddColumn( final String database, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index a530759adafd..d087aaea221d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -285,6 +285,13 @@ SettableFuture createTable( SettableFuture showTables(final String database); + SettableFuture alterTableRenameTable( + final String database, + final String sourceName, + final String targetName, + final String queryId, + final boolean tableIfExists); + SettableFuture alterTableAddColumn( final String database, final String tableName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameTableTask.java new file mode 100644 index 000000000000..78382e228385 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameTableTask.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +public class AlterTableRenameTableTask extends AbstractAlterTableTask { + + private final String targetName; + + public AlterTableRenameTableTask( + final String database, + final String tableName, + final String queryId, + final String targetName, + final boolean tableIfExists) { + super(database, tableName, queryId, tableIfExists); + this.targetName = targetName; + } + + @Override + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.alterTableRenameTable( + database, tableName, targetName, queryId, tableIfExists); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RenameTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RenameTable.java index 06d83cc36ec0..7385271fc7ed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RenameTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RenameTable.java @@ -31,16 +31,25 @@ public class RenameTable extends Statement { private final QualifiedName source; private final Identifier target; - public RenameTable(QualifiedName source, Identifier target) { + private final boolean tableIfExists; + + public RenameTable( + final QualifiedName source, final Identifier target, final boolean tableIfExists) { super(null); this.source = requireNonNull(source, "source name is null"); this.target = requireNonNull(target, "target name is null"); + this.tableIfExists = tableIfExists; } - public RenameTable(NodeLocation location, QualifiedName source, Identifier target) { + public RenameTable( + final NodeLocation location, + final QualifiedName source, + final Identifier target, + final boolean tableIfExists) { super(requireNonNull(location, "location is null")); this.source = requireNonNull(source, "source name is null"); this.target = requireNonNull(target, "target name is null"); + this.tableIfExists = tableIfExists; } public QualifiedName getSource() { @@ -51,8 +60,12 @@ public Identifier getTarget() { return target; } + public boolean tableIfExists() { + return tableIfExists; + } + @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, C context) { return visitor.visitRenameTable(this, context); } @@ -63,23 +76,29 @@ public List getChildren() { @Override public int hashCode() { - return Objects.hash(source, target); + return Objects.hash(source, target, tableIfExists); } @Override - public boolean equals(Object obj) { - if (this == obj) { + public boolean equals(final Object o) { + if (this == o) { return true; } - if ((obj == null) || (getClass() != obj.getClass())) { + if ((o == null) || (getClass() != o.getClass())) { return false; } - RenameTable o = (RenameTable) obj; - return Objects.equals(source, o.source) && Objects.equals(target, o.target); + final RenameTable that = (RenameTable) o; + return tableIfExists == that.tableIfExists + && Objects.equals(source, that.source) + && Objects.equals(target, that.target); } @Override public String toString() { - return toStringHelper(this).add("source", source).add("target", target).toString(); + return toStringHelper(this) + .add("source", source) + .add("target", target) + .add("tableIfExists", tableIfExists) + .toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index d8bd43c443e8..14ced5dc9ec9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -308,7 +308,10 @@ public Node visitDescTableStatement(RelationalSqlParser.DescTableStatementContex @Override public Node visitRenameTable(RelationalSqlParser.RenameTableContext ctx) { return new RenameTable( - getLocation(ctx), getQualifiedName(ctx.from), lowerIdentifier((Identifier) visit(ctx.to))); + getLocation(ctx), + getQualifiedName(ctx.from), + lowerIdentifier((Identifier) visit(ctx.to)), + Objects.nonNull(ctx.EXISTS())); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java index 778e03fe422a..450c73d44b7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java @@ -591,8 +591,12 @@ protected Void visitDropTable(DropTable node, Integer indent) { } @Override - protected Void visitRenameTable(RenameTable node, Integer indent) { + protected Void visitRenameTable(final RenameTable node, final Integer indent) { builder.append("ALTER TABLE "); + if (node.tableIfExists()) { + builder.append("IF EXISTS "); + } + builder .append(formatName(node.getSource())) .append(" RENAME TO ") diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java index 1b2345a4b5a8..4692d2acdad7 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java @@ -23,7 +23,8 @@ public enum AlterTableOperationType { ADD_COLUMN((byte) 0), SET_PROPERTIES((byte) 1), RENAME_COLUMN((byte) 2), - DROP_COLUMN((byte) 3); + DROP_COLUMN((byte) 3), + RENAME_TABLE((byte) 4); private final byte type; @@ -41,6 +42,12 @@ public static AlterTableOperationType getType(final byte value) { return ADD_COLUMN; case 1: return SET_PROPERTIES; + case 2: + return RENAME_COLUMN; + case 3: + return DROP_COLUMN; + case 4: + return RENAME_TABLE; default: throw new IllegalArgumentException(); } From f32a8422277e061202fab1afd6b16d587858ca5a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 26 Aug 2024 19:09:02 +0800 Subject: [PATCH 017/109] enrich --- .../table/RenameTableColumnProcedure.java | 46 +++++++++++++++++-- .../state/schema/RenameTableColumnState.java | 8 +++- .../config/TableConfigTaskVisitor.java | 6 +++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index 18e0c1d5dc31..7e9242cf6396 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -27,14 +27,20 @@ import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Objects; +import static org.apache.iotdb.confignode.procedure.state.schema.RenameTableColumnState.COMMIT_RELEASE; + public class RenameTableColumnProcedure extends AbstractAlterTableProcedure { + private static final Logger LOGGER = LoggerFactory.getLogger(RenameTableColumnProcedure.class); + private String oldName; private String newName; @@ -55,10 +61,44 @@ public RenameTableColumnProcedure( @Override protected Flow executeFromState( - final ConfigNodeProcedureEnv configNodeProcedureEnv, - final RenameTableColumnState renameTableColumnState) + final ConfigNodeProcedureEnv env, final RenameTableColumnState state) throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { - return null; + final long startTime = System.currentTimeMillis(); + try { + switch (state) { + case COLUMN_CHECK: + LOGGER.info("Column check for table {}.{} when renaming column", database, tableName); + columnCheck(env); + break; + case PRE_RELEASE: + LOGGER.info("Pre release info of table {}.{} when renaming column", database, tableName); + preRelease(env); + break; + case RENAME_COLUMN_ON_SCHEMA_REGION: + LOGGER.info("Rename column to table {}.{} on schema region", database, tableName); + addColumn(env); + break; + case RENAME_COLUMN_ON_CONFIG_NODE: + LOGGER.info("Rename column to table {}.{} on config node", database, tableName); + addColumn(env); + break; + case COMMIT_RELEASE: + LOGGER.info("Commit release info of table {}.{} when adding column", database, tableName); + commitRelease(env); + return Flow.NO_MORE_STATE; + default: + setFailure(new ProcedureException("Unrecognized AddTableColumnState " + state)); + return Flow.NO_MORE_STATE; + } + return Flow.HAS_MORE_STATE; + } finally { + LOGGER.info( + "AddTableColumn-{}.{}-{} costs {}ms", + database, + tableName, + state, + (System.currentTimeMillis() - startTime)); + } } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/RenameTableColumnState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/RenameTableColumnState.java index 58f34283199a..f8e52e58f96c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/RenameTableColumnState.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/RenameTableColumnState.java @@ -19,6 +19,10 @@ package org.apache.iotdb.confignode.procedure.state.schema; -public class RenameTableColumnState { - // TODO +public enum RenameTableColumnState { + COLUMN_CHECK, + PRE_RELEASE, + RENAME_COLUMN_ON_SCHEMA_REGION, + RENAME_COLUMN_ON_CONFIG_NODE, + COMMIT_RELEASE } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 3daef1fbefbb..a0a11163fd79 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -235,6 +235,12 @@ protected IConfigTask visitRenameColumn(final RenameColumn node, final MPPQueryC context.setQueryType(QueryType.WRITE); final Pair databaseTablePair = splitQualifiedName(node.getTable()); + final String oldName = node.getSource().getValue(); + final String newName = node.getTarget().getValue(); + if (oldName.equals(newName)) { + throw new SemanticException("The old column name shall not be equal to the new columnName."); + } + return new AlterTableRenameColumnTask( databaseTablePair.getLeft(), databaseTablePair.getRight(), From 6cc2b5958350f7556b03fcf229df754df3d374cb Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 26 Aug 2024 19:11:12 +0800 Subject: [PATCH 018/109] Update TableConfigTaskVisitor.java --- .../plan/execution/config/TableConfigTaskVisitor.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index a0a11163fd79..fa344685366f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -203,6 +203,12 @@ protected IConfigTask visitRenameTable(final RenameTable node, final MPPQueryCon context.setQueryType(QueryType.WRITE); final Pair databaseTablePair = splitQualifiedName(node.getSource()); + final String oldName = databaseTablePair.getRight(); + final String newName = node.getTarget().getValue(); + if (oldName.equals(newName)) { + throw new SemanticException("The table's old name shall not be equal to the new one."); + } + return new AlterTableRenameTableTask( databaseTablePair.getLeft(), databaseTablePair.getRight(), @@ -238,7 +244,7 @@ protected IConfigTask visitRenameColumn(final RenameColumn node, final MPPQueryC final String oldName = node.getSource().getValue(); final String newName = node.getTarget().getValue(); if (oldName.equals(newName)) { - throw new SemanticException("The old column name shall not be equal to the new columnName."); + throw new SemanticException("The column's old name shall not be equal to the new one."); } return new AlterTableRenameColumnTask( From b6a2a8dfdd74f7c21e326c4501d1a9f1d3d9501b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:41:41 +0800 Subject: [PATCH 019/109] continue --- .../org/apache/iotdb/rpc/TSStatusCode.java | 1 - .../manager/schema/ClusterSchemaManager.java | 39 +++++++++++++++++++ .../table/RenameTableColumnProcedure.java | 20 ++++++++++ .../iotdb/commons/schema/table/TsTable.java | 14 ++++++- 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java index 8b524a5f9795..4964ff4c06bd 100644 --- a/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java +++ b/iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java @@ -90,7 +90,6 @@ public enum TSStatusCode { TABLE_NOT_EXISTS(550), TABLE_ALREADY_EXISTS(551), COLUMN_ALREADY_EXISTS(552), - COLUMN_NOT_EXISTS(553), ONLY_LOGICAL_VIEW(560), // Storage Engine diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 49f1b92f4347..30cf230fb733 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -28,6 +28,7 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; +import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.utils.PathUtils; @@ -1143,6 +1144,44 @@ public synchronized Pair tableColumnCheckForColumnExtension( return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable); } + public synchronized Pair tableColumnCheckForColumnRenaming( + final String database, final String tableName, final String oldName, final String newName) { + final TsTable originalTable = getTableIfExists(database, tableName).orElse(null); + + if (Objects.isNull(originalTable)) { + return new Pair<>( + RpcUtils.getStatus( + TSStatusCode.TABLE_NOT_EXISTS, + String.format( + "Table '%s.%s' does not exist", + database.substring(ROOT.length() + 1), tableName)), + null); + } + + final TsTable expandedTable = TsTable.deserialize(ByteBuffer.wrap(originalTable.serialize())); + + final TsTableColumnSchema schema = originalTable.getColumnSchema(oldName); + if (Objects.isNull(schema)) { + return new Pair<>( + RpcUtils.getStatus( + TSStatusCode.COLUMN_NOT_EXISTS, String.format("Column '%s' does not exist", oldName)), + null); + } + + if (schema.getColumnCategory() != TsTableColumnCategory.ATTRIBUTE) { + return new Pair<>( + RpcUtils.getStatus( + TSStatusCode.COLUMN_CATEGORY_MISMATCH, + "Currently we only support renaming for attribute columns, current category is " + + schema.getColumnCategory()), + null); + } + + expandedTable.renameColumnSchema(oldName, newName); + + return new Pair<>(RpcUtils.SUCCESS_STATUS, expandedTable); + } + public synchronized TSStatus addTableColumn( final String database, final String tableName, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index 7e9242cf6396..5870518c19de 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -19,13 +19,19 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; +import org.apache.iotdb.confignode.procedure.state.schema.AddTableColumnState; import org.apache.iotdb.confignode.procedure.state.schema.RenameTableColumnState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -101,6 +107,20 @@ protected Flow executeFromState( } } + private void columnCheck(final ConfigNodeProcedureEnv env) { + final Pair result = + env.getConfigManager() + .getClusterSchemaManager() + .tableColumnCheckForColumnRenaming(database, tableName, oldName, newName); + final TSStatus status = result.getLeft(); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + return; + } + table = result.getRight(); + setNextState(AddTableColumnState.PRE_RELEASE); + } + @Override protected void rollbackState( final ConfigNodeProcedureEnv configNodeProcedureEnv, diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 5d7cc60ab2fc..3dc57790548e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -104,10 +104,20 @@ public void addColumnSchema(final TsTableColumnSchema columnSchema) { } } - public void removeColumnSchema(String columnName) { + // Currently only supports attribute column + public void renameColumnSchema(final String oldName, final String newName) { readWriteLock.writeLock().lock(); try { - TsTableColumnSchema columnSchema = columnSchemaMap.remove(columnName); + columnSchemaMap.put(newName, columnSchemaMap.remove(oldName)); + } finally { + readWriteLock.writeLock().unlock(); + } + } + + public void removeColumnSchema(final String columnName) { + readWriteLock.writeLock().lock(); + try { + final TsTableColumnSchema columnSchema = columnSchemaMap.remove(columnName); if (columnSchema != null && columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { idNums--; From c3f8999a1b90a0ebd6121a9bad4ca46daac98602 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:45:52 +0800 Subject: [PATCH 020/109] Update RenameTableColumnProcedure.java --- .../impl/schema/table/RenameTableColumnProcedure.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index 5870518c19de..2c7277f11fef 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -26,7 +26,6 @@ import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; -import org.apache.iotdb.confignode.procedure.state.schema.AddTableColumnState; import org.apache.iotdb.confignode.procedure.state.schema.RenameTableColumnState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.rpc.TSStatusCode; @@ -118,7 +117,7 @@ private void columnCheck(final ConfigNodeProcedureEnv env) { return; } table = result.getRight(); - setNextState(AddTableColumnState.PRE_RELEASE); + setNextState(RenameTableColumnState.PRE_RELEASE); } @Override @@ -129,17 +128,17 @@ protected void rollbackState( @Override protected RenameTableColumnState getState(final int stateId) { - return null; + return RenameTableColumnState.values()[stateId]; } @Override - protected int getStateId(final RenameTableColumnState renameTableColumnState) { - return 0; + protected int getStateId(final RenameTableColumnState state) { + return state.ordinal(); } @Override protected RenameTableColumnState getInitialState() { - return null; + return RenameTableColumnState.COLUMN_CHECK; } @Override From ab1676d388834b14e961f02321c295565dbb4867 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 27 Aug 2024 10:56:31 +0800 Subject: [PATCH 021/109] Added plan --- .../request/ConfigPhysicalPlanType.java | 1 + .../write/table/RenameTableColumnPlan.java | 86 +++++++++++++++++++ .../manager/schema/ClusterSchemaManager.java | 14 +++ 3 files changed, 101 insertions(+) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RenameTableColumnPlan.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 8defa85b9738..0d3130068c1c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -160,6 +160,7 @@ public enum ConfigPhysicalPlanType { CommitCreateTable((short) 852), AddTableColumn((short) 853), SetTableProperties((short) 854), + RenameTableColumn((short) 855), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RenameTableColumnPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RenameTableColumnPlan.java new file mode 100644 index 000000000000..71cfc20e6c5c --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RenameTableColumnPlan.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class RenameTableColumnPlan extends ConfigPhysicalPlan { + + private String database; + + private String tableName; + + private String oldName; + private String newName; + + public RenameTableColumnPlan() { + super(ConfigPhysicalPlanType.RenameTableColumn); + } + + public RenameTableColumnPlan( + final String database, final String tableName, final String oldName, final String newName) { + super(ConfigPhysicalPlanType.RenameTableColumn); + this.database = database; + this.tableName = tableName; + this.oldName = oldName; + this.newName = newName; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + public String getOldName() { + return oldName; + } + + public String getNewName() { + return newName; + } + + @Override + protected void serializeImpl(final DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + ReadWriteIOUtils.write(oldName, stream); + ReadWriteIOUtils.write(newName, stream); + } + + @Override + protected void deserializeImpl(final ByteBuffer buffer) throws IOException { + this.database = ReadWriteIOUtils.readString(buffer); + this.tableName = ReadWriteIOUtils.readString(buffer); + this.oldName = ReadWriteIOUtils.readString(buffer); + this.newName = ReadWriteIOUtils.readString(buffer); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 30cf230fb733..a8c546c0d554 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -1210,6 +1210,20 @@ public synchronized TSStatus rollbackAddTableColumn( } } + public synchronized TSStatus renameTableColumn( + final String database, + final String tableName, + final List columnSchemaList) { + final AddTableColumnPlan addTableColumnPlan = + new AddTableColumnPlan(database, tableName, columnSchemaList, false); + try { + return getConsensusManager().write(addTableColumnPlan); + } catch (final ConsensusException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR, e.getMessage()); + } + } + public synchronized TSStatus setTableProperties( final String database, final String tableName, final Map properties) { final SetTablePropertiesPlan setTablePropertiesPlan = From 4fb0ac63382a9847a06f07fa917b6d50ead59b0d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:04:30 +0800 Subject: [PATCH 022/109] Update ConfigPhysicalPlanSerDeTest.java --- .../request/ConfigPhysicalPlanSerDeTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 86fde2710a0a..a1ad8e892662 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -149,6 +149,7 @@ import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.SetTablePropertiesPlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; @@ -1524,6 +1525,23 @@ public void RollbackCreateTablePlanTest() throws IOException { rollbackCreateTablePlan0.getTableName(), rollbackCreateTablePlan1.getTableName()); } + @Test + public void RenameTableColumnPlan() throws IOException { + final RenameTableColumnPlan renameTablePropertiesPlan0 = + new RenameTableColumnPlan("root.database1", "table1", "attr1", "att2"); + final RenameTableColumnPlan renameTablePropertiesPlan1 = + (RenameTableColumnPlan) + ConfigPhysicalPlan.Factory.create(renameTablePropertiesPlan0.serializeToByteBuffer()); + Assert.assertEquals( + renameTablePropertiesPlan0.getDatabase(), renameTablePropertiesPlan1.getDatabase()); + Assert.assertEquals( + renameTablePropertiesPlan0.getTableName(), renameTablePropertiesPlan1.getTableName()); + Assert.assertEquals( + renameTablePropertiesPlan0.getOldName(), renameTablePropertiesPlan1.getOldName()); + Assert.assertEquals( + renameTablePropertiesPlan0.getNewName(), renameTablePropertiesPlan1.getNewName()); + } + @Test public void SetTablePropertiesPlanTest() throws IOException { final SetTablePropertiesPlan setTablePropertiesPlan0 = From b9e8797a8c2205193b184614603ed9c3c13d65e8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:25:17 +0800 Subject: [PATCH 023/109] Added configNode logic --- .../consensus/request/ConfigPhysicalPlan.java | 4 ++ .../manager/schema/ClusterSchemaManager.java | 11 ++-- .../executor/ConfigPlanExecutor.java | 3 + .../persistence/schema/ClusterSchemaInfo.java | 16 +++++ .../persistence/schema/ConfigMTree.java | 9 +++ .../schema/table/AddTableColumnProcedure.java | 5 +- .../table/RenameTableColumnProcedure.java | 58 ++++++++++++++++--- 7 files changed, 90 insertions(+), 16 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index 00e7345f650c..4a851f4c88e4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -118,6 +118,7 @@ import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.SetTablePropertiesPlan; import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; @@ -416,6 +417,9 @@ public static ConfigPhysicalPlan create(ByteBuffer buffer) throws IOException { case SetTableProperties: plan = new SetTablePropertiesPlan(); break; + case RenameTableColumn: + plan = new RenameTableColumnPlan(); + break; case GetNodePathsPartition: plan = new GetNodePathsPartitionPlan(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index a8c546c0d554..bd07630c3564 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -53,6 +53,7 @@ import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.payload.PipeEnrichedPlan; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.SetTablePropertiesPlan; import org.apache.iotdb.confignode.consensus.request.write.template.CreateSchemaTemplatePlan; import org.apache.iotdb.confignode.consensus.request.write.template.DropSchemaTemplatePlan; @@ -1211,13 +1212,11 @@ public synchronized TSStatus rollbackAddTableColumn( } public synchronized TSStatus renameTableColumn( - final String database, - final String tableName, - final List columnSchemaList) { - final AddTableColumnPlan addTableColumnPlan = - new AddTableColumnPlan(database, tableName, columnSchemaList, false); + final String database, final String tableName, final String oldName, final String newName) { + final RenameTableColumnPlan renameTableColumnPlan = + new RenameTableColumnPlan(database, tableName, oldName, newName); try { - return getConsensusManager().write(addTableColumnPlan); + return getConsensusManager().write(renameTableColumnPlan); } catch (final ConsensusException e) { LOGGER.warn(e.getMessage(), e); return RpcUtils.getStatus(TSStatusCode.INTERNAL_SERVER_ERROR, e.getMessage()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index 1b3a6488ef73..816713dfd028 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -112,6 +112,7 @@ import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.SetTablePropertiesPlan; import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; @@ -501,6 +502,8 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) return clusterSchemaInfo.commitCreateTable((CommitCreateTablePlan) physicalPlan); case AddTableColumn: return clusterSchemaInfo.addTableColumn((AddTableColumnPlan) physicalPlan); + case RenameTableColumn: + return clusterSchemaInfo.renameTableColumn((RenameTableColumnPlan) physicalPlan); case SetTableProperties: return clusterSchemaInfo.setTableProperties((SetTablePropertiesPlan) physicalPlan); case CreatePipeV2: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 2d57c1731b5a..679e9255cebe 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -48,6 +48,7 @@ import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.SetTablePropertiesPlan; import org.apache.iotdb.confignode.consensus.request.write.template.CommitSetSchemaTemplatePlan; @@ -1152,6 +1153,21 @@ public TSStatus addTableColumn(final AddTableColumnPlan plan) { } } + public TSStatus renameTableColumn(final RenameTableColumnPlan plan) { + final String databaseName = PathUtils.qualifyDatabaseName(plan.getDatabase()); + databaseReadWriteLock.writeLock().lock(); + try { + mTree.renameTableColumn( + new PartialPath(databaseName), plan.getTableName(), plan.getOldName(), plan.getNewName()); + return RpcUtils.SUCCESS_STATUS; + } catch (final MetadataException e) { + LOGGER.warn(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + public TSStatus setTableProperties(final SetTablePropertiesPlan plan) { final String databaseName = PathUtils.qualifyDatabaseName(plan.getDatabase()); databaseReadWriteLock.writeLock().lock(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index c7a2af3a27f3..647bff00fc5e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -738,6 +738,15 @@ public void rollbackAddTableColumn( columnSchemaList.forEach(o -> table.removeColumnSchema(o.getColumnName())); } + public void renameTableColumn( + final PartialPath database, + final String tableName, + final String oldName, + final String newName) + throws MetadataException { + getTable(database, tableName).renameColumnSchema(oldName, newName); + } + public void setTableProperties( final PartialPath database, final String tableName, final Map tableProperties) throws MetadataException { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index 62a39b211731..30d17c44b5c6 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -140,13 +140,14 @@ protected void rollbackState(final ConfigNodeProcedureEnv env, final AddTableCol switch (state) { case ADD_COLUMN: LOGGER.info( - "Start rollback pre release info of table {}.{} when adding column", + "Start rollback Add column to table {}.{} when adding column", database, table.getTableName()); rollbackAddColumn(env); break; case PRE_RELEASE: - LOGGER.info("Start rollback Add column to table {}.{}", database, table.getTableName()); + LOGGER.info( + "Start rollback pre release info of table {}.{}", database, table.getTableName()); rollbackPreRelease(env); break; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index 2c7277f11fef..e5ad24c8a32b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -40,8 +40,6 @@ import java.nio.ByteBuffer; import java.util.Objects; -import static org.apache.iotdb.confignode.procedure.state.schema.RenameTableColumnState.COMMIT_RELEASE; - public class RenameTableColumnProcedure extends AbstractAlterTableProcedure { private static final Logger LOGGER = LoggerFactory.getLogger(RenameTableColumnProcedure.class); @@ -81,11 +79,11 @@ protected Flow executeFromState( break; case RENAME_COLUMN_ON_SCHEMA_REGION: LOGGER.info("Rename column to table {}.{} on schema region", database, tableName); - addColumn(env); + // TODO break; case RENAME_COLUMN_ON_CONFIG_NODE: LOGGER.info("Rename column to table {}.{} on config node", database, tableName); - addColumn(env); + renameColumn(env); break; case COMMIT_RELEASE: LOGGER.info("Commit release info of table {}.{} when adding column", database, tableName); @@ -120,11 +118,55 @@ private void columnCheck(final ConfigNodeProcedureEnv env) { setNextState(RenameTableColumnState.PRE_RELEASE); } + private void renameColumn(final ConfigNodeProcedureEnv env) { + final TSStatus status = + env.getConfigManager() + .getClusterSchemaManager() + .renameTableColumn(database, tableName, oldName, newName); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } else { + setNextState(RenameTableColumnState.COMMIT_RELEASE); + } + } + @Override - protected void rollbackState( - final ConfigNodeProcedureEnv configNodeProcedureEnv, - final RenameTableColumnState renameTableColumnState) - throws IOException, InterruptedException, ProcedureException {} + protected void rollbackState(final ConfigNodeProcedureEnv env, final RenameTableColumnState state) + throws IOException, InterruptedException, ProcedureException { + final long startTime = System.currentTimeMillis(); + try { + switch (state) { + case RENAME_COLUMN_ON_CONFIG_NODE: + LOGGER.info( + "Start rollback Renaming column to table {}.{} on configNode", + database, + table.getTableName()); + rollbackRenameColumn(env); + break; + case PRE_RELEASE: + LOGGER.info( + "Start rollback pre release info of table {}.{}", database, table.getTableName()); + rollbackPreRelease(env); + break; + } + } finally { + LOGGER.info( + "Rollback DropTable-{} costs {}ms.", state, (System.currentTimeMillis() - startTime)); + } + } + + private void rollbackRenameColumn(final ConfigNodeProcedureEnv env) { + if (table == null) { + return; + } + final TSStatus status = + env.getConfigManager() + .getClusterSchemaManager() + .renameTableColumn(database, tableName, newName, oldName); + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + } @Override protected RenameTableColumnState getState(final int stateId) { From 13a27bda950fbd318e2dc86aa28608fbcfb62bbc Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:39:27 +0800 Subject: [PATCH 024/109] Idempotency --- .../schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 2 ++ .../java/org/apache/iotdb/commons/schema/table/TsTable.java | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 9ba87e08612a..9ec54a419e3a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -1571,6 +1571,8 @@ protected void updateEntity(final IDeviceMNode node) { } } + public void renameTableAttribute() {} + public void deleteTableDevice(String tableName) { storageGroupMNode.deleteChild(tableName); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java index 3dc57790548e..0bdee43f1257 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TsTable.java @@ -108,7 +108,10 @@ public void addColumnSchema(final TsTableColumnSchema columnSchema) { public void renameColumnSchema(final String oldName, final String newName) { readWriteLock.writeLock().lock(); try { - columnSchemaMap.put(newName, columnSchemaMap.remove(oldName)); + // Ensures idempotency + if (columnSchemaMap.containsKey(oldName)) { + columnSchemaMap.put(newName, columnSchemaMap.remove(oldName)); + } } finally { readWriteLock.writeLock().unlock(); } From 884d8695e2eb7afd12f29b83c0481400dbb17af1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 12 Oct 2024 20:53:50 +0800 Subject: [PATCH 025/109] Renaming --- ...tAlterTableTask.java => AbstractAlterOrDropTableTask.java} | 4 ++-- .../config/metadata/relational/AlterTableAddColumnTask.java | 2 +- .../config/metadata/relational/AlterTableDropColumnTask.java | 2 +- .../metadata/relational/AlterTableRenameColumnTask.java | 2 +- .../config/metadata/relational/AlterTableRenameTableTask.java | 2 +- .../metadata/relational/AlterTableSetPropertiesTask.java | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/{AbstractAlterTableTask.java => AbstractAlterOrDropTableTask.java} (92%) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractAlterTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractAlterOrDropTableTask.java similarity index 92% rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractAlterTableTask.java rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractAlterOrDropTableTask.java index 6fc1ef752cb3..9be925e04258 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractAlterTableTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AbstractAlterOrDropTableTask.java @@ -22,7 +22,7 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; -abstract class AbstractAlterTableTask implements IConfigTask { +abstract class AbstractAlterOrDropTableTask implements IConfigTask { protected final String database; @@ -32,7 +32,7 @@ abstract class AbstractAlterTableTask implements IConfigTask { protected final boolean tableIfExists; - protected AbstractAlterTableTask( + protected AbstractAlterOrDropTableTask( String database, final String tableName, final String queryId, final boolean tableIfExists) { database = PathUtils.qualifyDatabaseName(database); this.database = database; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java index 20269554872b..944f7bb2206b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableAddColumnTask.java @@ -27,7 +27,7 @@ import java.util.List; -public class AlterTableAddColumnTask extends AbstractAlterTableTask { +public class AlterTableAddColumnTask extends AbstractAlterOrDropTableTask { private final List columnList; private final boolean columnIfExists; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java index dc136570f53e..67135608322f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableDropColumnTask.java @@ -24,7 +24,7 @@ import com.google.common.util.concurrent.ListenableFuture; -public class AlterTableDropColumnTask extends AbstractAlterTableTask { +public class AlterTableDropColumnTask extends AbstractAlterOrDropTableTask { private final String columnName; private final boolean columnIfExists; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java index 9caee19c96dd..8b0df2ef10eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameColumnTask.java @@ -24,7 +24,7 @@ import com.google.common.util.concurrent.ListenableFuture; -public class AlterTableRenameColumnTask extends AbstractAlterTableTask { +public class AlterTableRenameColumnTask extends AbstractAlterOrDropTableTask { private final String oldName; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameTableTask.java index 78382e228385..f26e734397fb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameTableTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableRenameTableTask.java @@ -24,7 +24,7 @@ import com.google.common.util.concurrent.ListenableFuture; -public class AlterTableRenameTableTask extends AbstractAlterTableTask { +public class AlterTableRenameTableTask extends AbstractAlterOrDropTableTask { private final String targetName; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableSetPropertiesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableSetPropertiesTask.java index cf927eb625db..d4c19439bf9e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableSetPropertiesTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/AlterTableSetPropertiesTask.java @@ -26,7 +26,7 @@ import java.util.Map; -public class AlterTableSetPropertiesTask extends AbstractAlterTableTask { +public class AlterTableSetPropertiesTask extends AbstractAlterOrDropTableTask { private final Map properties; From e71bfdc425321f02d00fc2160a9c9d5cdfea34b1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 12 Oct 2024 20:57:26 +0800 Subject: [PATCH 026/109] Update AbstractAlterTableProcedure.java --- .../procedure/impl/schema/table/AbstractAlterTableProcedure.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java index 236cb898b9a1..85bb3f33c5ec 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java @@ -96,7 +96,6 @@ protected void commitRelease(final ConfigNodeProcedureEnv env) { database, table.getTableName(), failedResults); - // TODO: Handle commit failure } } From f97d1c67c17d411a018b7ee0a065692ac83db6a5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Sat, 12 Oct 2024 20:38:01 +0800 Subject: [PATCH 027/109] Partial --- .../execution/config/TableConfigTaskVisitor.java | 4 +++- .../plan/relational/sql/ast/DropTable.java | 11 ++++++----- .../plan/relational/sql/parser/AstBuilder.java | 16 +++++++--------- .../plan/relational/sql/util/SqlFormatter.java | 2 +- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 20d9c15dce37..5bda3e94eb1e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -440,7 +440,9 @@ private TSDataType getDataType(final DataType dataType) { } @Override - protected IConfigTask visitDropTable(DropTable node, MPPQueryContext context) { + protected IConfigTask visitDropTable(final DropTable node, final MPPQueryContext context) { + context.setQueryType(QueryType.WRITE); + final Pair databaseTablePair = splitQualifiedName(node.getTableName()); return super.visitDropTable(node, context); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropTable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropTable.java index 13cc623dd5af..54a04f472fb4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropTable.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropTable.java @@ -32,13 +32,14 @@ public class DropTable extends Statement { private final QualifiedName tableName; private final boolean exists; - public DropTable(QualifiedName tableName, boolean exists) { + public DropTable(final QualifiedName tableName, final boolean exists) { super(null); this.tableName = requireNonNull(tableName, "tableName is null"); this.exists = exists; } - public DropTable(NodeLocation location, QualifiedName tableName, boolean exists) { + public DropTable( + final NodeLocation location, final QualifiedName tableName, final boolean exists) { super(requireNonNull(location, "location is null")); this.tableName = requireNonNull(tableName, "tableName is null"); this.exists = exists; @@ -53,7 +54,7 @@ public boolean isExists() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitDropTable(this, context); } @@ -68,14 +69,14 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if ((obj == null) || (getClass() != obj.getClass())) { return false; } - DropTable o = (DropTable) obj; + final DropTable o = (DropTable) obj; return Objects.equals(tableName, o.tableName) && (exists == o.exists); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index e790e2b1a631..68c8f719dedf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -301,27 +301,25 @@ public Node visitColumnDefinition(final RelationalSqlParser.ColumnDefinitionCont } @Override - public Node visitDropTableStatement(RelationalSqlParser.DropTableStatementContext ctx) { + public Node visitDropTableStatement(final RelationalSqlParser.DropTableStatementContext ctx) { return new DropTable( getLocation(ctx), getQualifiedName(ctx.qualifiedName()), ctx.EXISTS() != null); } @Override - public Node visitShowTableStatement(RelationalSqlParser.ShowTableStatementContext ctx) { - if (ctx.database == null) { - return new ShowTables(getLocation(ctx)); - } else { - return new ShowTables(getLocation(ctx), lowerIdentifier((Identifier) visit(ctx.database))); - } + public Node visitShowTableStatement(final RelationalSqlParser.ShowTableStatementContext ctx) { + return Objects.nonNull(ctx.database) + ? new ShowTables(getLocation(ctx), lowerIdentifier((Identifier) visit(ctx.database))) + : new ShowTables(getLocation(ctx)); } @Override - public Node visitDescTableStatement(RelationalSqlParser.DescTableStatementContext ctx) { + public Node visitDescTableStatement(final RelationalSqlParser.DescTableStatementContext ctx) { return new DescribeTable(getLocation(ctx), getQualifiedName(ctx.table)); } @Override - public Node visitRenameTable(RelationalSqlParser.RenameTableContext ctx) { + public Node visitRenameTable(final RelationalSqlParser.RenameTableContext ctx) { return new RenameTable( getLocation(ctx), getQualifiedName(ctx.from), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java index 424cd28e9872..5a015ce06a63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java @@ -629,7 +629,7 @@ private String formatColumnDefinition(ColumnDefinition column) { } @Override - protected Void visitDropTable(DropTable node, Integer indent) { + protected Void visitDropTable(final DropTable node, final Integer indent) { builder.append("DROP TABLE "); if (node.isExists()) { builder.append("IF EXISTS "); From 83d8dc0bde31307c9aaa9aeb9db7261eb2b31ef1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:58:25 +0800 Subject: [PATCH 028/109] partial --- .../config/TableConfigTaskVisitor.java | 7 +++- .../metadata/relational/DropTableTask.java | 42 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 5bda3e94eb1e..7df9f5c3ba69 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.CreateTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DescribeTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DropDBTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DropTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowAINodesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowConfigNodesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowDBTask; @@ -443,7 +444,11 @@ private TSDataType getDataType(final DataType dataType) { protected IConfigTask visitDropTable(final DropTable node, final MPPQueryContext context) { context.setQueryType(QueryType.WRITE); final Pair databaseTablePair = splitQualifiedName(node.getTableName()); - return super.visitDropTable(node, context); + return new DropTableTask( + databaseTablePair.getLeft(), + databaseTablePair.getRight(), + context.getQueryId().getId(), + node.isExists()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java new file mode 100644 index 000000000000..fb5f610a36f7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +public class DropTableTask extends AbstractAlterOrDropTableTask { + + public DropTableTask( + final String database, + final String tableName, + final String queryId, + final boolean tableIfExists) { + super(database, tableName, queryId, tableIfExists); + } + + @Override + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return null; + } +} From 76320eabf2cf5abaadc2b69af51895c94f5a66cd Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:27:20 +0800 Subject: [PATCH 029/109] Update TableDeviceFetchSource.java --- .../schema/source/TableDeviceFetchSource.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java index bf13dc4ba974..afba466bf046 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java @@ -35,6 +35,7 @@ import org.apache.tsfile.utils.Binary; import java.util.List; +import java.util.Objects; public class TableDeviceFetchSource implements ISchemaSource { @@ -73,14 +74,14 @@ public List getInfoQueryColumnHeaders() { @Override public void transformToTsBlockColumns( - IDeviceSchemaInfo schemaInfo, TsBlockBuilder builder, String database) { + final IDeviceSchemaInfo schemaInfo, final TsBlockBuilder builder, final String database) { builder.getTimeColumnBuilder().writeLong(0L); int resultIndex = 0; int idIndex = 0; - String[] pathNodes = schemaInfo.getRawNodes(); - TsTable table = DataNodeTableCache.getInstance().getTable(this.database, tableName); + final String[] pathNodes = schemaInfo.getRawNodes(); + final TsTable table = DataNodeTableCache.getInstance().getTable(this.database, tableName); TsTableColumnSchema columnSchema; - for (ColumnHeader columnHeader : columnHeaderList) { + for (final ColumnHeader columnHeader : columnHeaderList) { columnSchema = table.getColumnSchema(columnHeader.getColumnName()); if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ID)) { if (pathNodes.length <= idIndex + 3 || pathNodes[idIndex + 3] == null) { @@ -92,8 +93,7 @@ public void transformToTsBlockColumns( } idIndex++; } else if (columnSchema.getColumnCategory().equals(TsTableColumnCategory.ATTRIBUTE)) { - String attributeValue = schemaInfo.getAttributeValue(columnHeader.getColumnName()); - if (attributeValue == null) { + if (Objects.isNull(schemaInfo.getAttributeValue(columnHeader.getColumnName()))) { builder.getColumnBuilder(resultIndex).appendNull(); } else { builder From 5b84aa5ccbef9f73e6b3439c3fb6408834676f7d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:30:15 +0800 Subject: [PATCH 030/109] Refactor --- .../operator/schema/source/ISchemaSource.java | 21 +++++++++------- .../schema/source/TableDeviceFetchSource.java | 24 +++++++++---------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java index 2a80e1071e53..c71b2cb6db73 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/ISchemaSource.java @@ -21,9 +21,11 @@ import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegion; +import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegion; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.info.ISchemaInfo; import org.apache.iotdb.db.schemaengine.schemaregion.read.resp.reader.ISchemaReader; +import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.block.TsBlockBuilder; import java.util.List; @@ -31,25 +33,26 @@ public interface ISchemaSource { /** - * Get the SchemaReader for iterating target SchemaInfo from given SchemaRegion. + * Get the {@link ISchemaReader} for iterating target SchemaInfo from given {@link SchemaRegion}. * - * @return the SchemaReader for SchemaInfo T + * @return the {@link ISchemaReader} for SchemaInfo T */ - ISchemaReader getSchemaReader(ISchemaRegion schemaRegion); + ISchemaReader getSchemaReader(final ISchemaRegion schemaRegion); - /** Get the column headers of the result TsBlock for SchemaInfo query. */ + /** Get the column headers of the result {@link TsBlock} for SchemaInfo query. */ List getInfoQueryColumnHeaders(); /** - * Transform the SchemaInfo T to fill the TsBlock. + * Transform the SchemaInfo T to fill the {@link TsBlock}. * * @param schemaInfo the SchemaInfo need to be processed - * @param tsBlockBuilder the target TsBlockBuilder using for generating TsBlock + * @param tsBlockBuilder the target {@link TsBlockBuilder} using for generating TsBlock * @param database the belonged databased of given SchemaInfo */ - void transformToTsBlockColumns(T schemaInfo, TsBlockBuilder tsBlockBuilder, String database); + void transformToTsBlockColumns( + final T schemaInfo, final TsBlockBuilder tsBlockBuilder, final String database); - boolean hasSchemaStatistic(ISchemaRegion schemaRegion); + boolean hasSchemaStatistic(final ISchemaRegion schemaRegion); - long getSchemaStatistic(ISchemaRegion schemaRegion); + long getSchemaStatistic(final ISchemaRegion schemaRegion); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java index afba466bf046..fc2580fd74c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TableDeviceFetchSource.java @@ -39,19 +39,19 @@ public class TableDeviceFetchSource implements ISchemaSource { - private String database; + private final String database; - private String tableName; + private final String tableName; - private List deviceIdList; + private final List deviceIdList; - private List columnHeaderList; + private final List columnHeaderList; public TableDeviceFetchSource( - String database, - String tableName, - List deviceIdList, - List columnHeaderList) { + final String database, + final String tableName, + final List deviceIdList, + final List columnHeaderList) { this.database = database; this.tableName = tableName; this.deviceIdList = deviceIdList; @@ -59,10 +59,10 @@ public TableDeviceFetchSource( } @Override - public ISchemaReader getSchemaReader(ISchemaRegion schemaRegion) { + public ISchemaReader getSchemaReader(final ISchemaRegion schemaRegion) { try { return schemaRegion.getTableDeviceReader(tableName, deviceIdList); - } catch (MetadataException e) { + } catch (final MetadataException e) { throw new SchemaExecutionException(e); } } @@ -110,12 +110,12 @@ public void transformToTsBlockColumns( } @Override - public boolean hasSchemaStatistic(ISchemaRegion schemaRegion) { + public boolean hasSchemaStatistic(final ISchemaRegion schemaRegion) { return false; } @Override - public long getSchemaStatistic(ISchemaRegion schemaRegion) { + public long getSchemaStatistic(final ISchemaRegion schemaRegion) { return 0; } } From 2c4c43eb6d1e90561fa5e5d383f30c64963c325f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:35:02 +0800 Subject: [PATCH 031/109] Add --- .../confignode/manager/ConfigManager.java | 4 +- .../executor/ClusterConfigTaskExecutor.java | 43 ++++++++++++++++--- .../config/executor/IConfigTaskExecutor.java | 3 ++ .../metadata/relational/DropDBTask.java | 4 +- ...ava => AlterOrDropTableOperationType.java} | 9 ++-- 5 files changed, 48 insertions(+), 15 deletions(-) rename iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/{AlterTableOperationType.java => AlterOrDropTableOperationType.java} (86%) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index ebccecdab5df..ee19a62edafa 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -49,7 +49,7 @@ import org.apache.iotdb.commons.path.PathPatternUtil; import org.apache.iotdb.commons.pipe.connector.payload.airgap.AirGapPseudoTPipeTransferRequest; import org.apache.iotdb.commons.schema.SchemaConstant; -import org.apache.iotdb.commons.schema.table.AlterTableOperationType; +import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.schema.ttl.TTLCache; @@ -2570,7 +2570,7 @@ public TSStatus createTable(final ByteBuffer tableInfo) { public TSStatus alterTable(final TAlterTableReq req) { final TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - switch (AlterTableOperationType.getType(req.operationType)) { + switch (AlterOrDropTableOperationType.getType(req.operationType)) { case ADD_COLUMN: return procedureManager.alterTableAddColumn(req); case SET_PROPERTIES: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 67548b24d27d..3d21c47a13c1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -47,7 +47,7 @@ import org.apache.iotdb.commons.pipe.agent.plugin.service.PipePluginExecutableManager; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; import org.apache.iotdb.commons.pipe.connector.payload.airgap.AirGapPseudoTPipeTransferRequest; -import org.apache.iotdb.commons.schema.table.AlterTableOperationType; +import org.apache.iotdb.commons.schema.table.AlterOrDropTableOperationType; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.TsTableInternalRPCUtil; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; @@ -3179,7 +3179,7 @@ public SettableFuture alterTableRenameTable( database, sourceName, queryId, - AlterTableOperationType.RENAME_TABLE, + AlterOrDropTableOperationType.RENAME_TABLE, stream.toByteArray(), client); @@ -3219,7 +3219,7 @@ public SettableFuture alterTableAddColumn( database, tableName, queryId, - AlterTableOperationType.ADD_COLUMN, + AlterOrDropTableOperationType.ADD_COLUMN, TsTableColumnSchemaUtil.serialize(columnSchemaList), client); @@ -3265,7 +3265,7 @@ public SettableFuture alterTableRenameColumn( database, tableName, queryId, - AlterTableOperationType.RENAME_COLUMN, + AlterOrDropTableOperationType.RENAME_COLUMN, stream.toByteArray(), client); @@ -3314,7 +3314,7 @@ public SettableFuture alterTableDropColumn( database, tableName, queryId, - AlterTableOperationType.DROP_COLUMN, + AlterOrDropTableOperationType.DROP_COLUMN, stream.toByteArray(), client); @@ -3361,7 +3361,7 @@ public SettableFuture alterTableSetProperties( database, tableName, queryId, - AlterTableOperationType.SET_PROPERTIES, + AlterOrDropTableOperationType.SET_PROPERTIES, stream.toByteArray(), client); @@ -3383,11 +3383,40 @@ public SettableFuture alterTableSetProperties( return future; } + @Override + public SettableFuture dropTable( + final String database, final String tableName, final String queryId, final boolean ifExists) { + final SettableFuture future = SettableFuture.create(); + try (final ConfigNodeClient client = + CLUSTER_DELETION_CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { + + final TSStatus tsStatus = + sendAlterReq2ConfigNode( + database, + tableName, + queryId, + AlterOrDropTableOperationType.DROP_TABLE, + new byte[0], + client); + + if (TSStatusCode.SUCCESS_STATUS.getStatusCode() == tsStatus.getCode() + || (TSStatusCode.TABLE_NOT_EXISTS.getStatusCode() == tsStatus.getCode() && ifExists)) { + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS)); + } else { + LOGGER.warn("Failed to drop table {}.{}, status is {}.", database, tableName, tsStatus); + future.setException(new IoTDBException(tsStatus.getMessage(), tsStatus.getCode())); + } + } catch (final ClientManagerException | TException e) { + future.setException(e); + } + return future; + } + private TSStatus sendAlterReq2ConfigNode( final String database, final String tableName, final String queryId, - final AlterTableOperationType type, + final AlterOrDropTableOperationType type, final byte[] updateInfo, final ConfigNodeClient client) throws ClientManagerException, TException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index 38288745b13b..fcb4c2adde25 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -329,4 +329,7 @@ SettableFuture alterTableSetProperties( final Map properties, final String queryId, final boolean ifExists); + + SettableFuture dropTable( + final String database, final String tableName, final String queryId, final boolean ifExists); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropDBTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropDBTask.java index d4740be37f71..1d6c5044ce8c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropDBTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropDBTask.java @@ -30,12 +30,12 @@ public class DropDBTask implements IConfigTask { private final DropDB node; - public DropDBTask(DropDB node) { + public DropDBTask(final DropDB node) { this.node = node; } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.dropDatabase(node); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterOrDropTableOperationType.java similarity index 86% rename from iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterOrDropTableOperationType.java index 4692d2acdad7..3a742bb20aa5 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterTableOperationType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterOrDropTableOperationType.java @@ -19,16 +19,17 @@ package org.apache.iotdb.commons.schema.table; -public enum AlterTableOperationType { +public enum AlterOrDropTableOperationType { ADD_COLUMN((byte) 0), SET_PROPERTIES((byte) 1), RENAME_COLUMN((byte) 2), DROP_COLUMN((byte) 3), - RENAME_TABLE((byte) 4); + RENAME_TABLE((byte) 4), + DROP_TABLE((byte) 5); private final byte type; - AlterTableOperationType(final byte type) { + AlterOrDropTableOperationType(final byte type) { this.type = type; } @@ -36,7 +37,7 @@ public byte getTypeValue() { return type; } - public static AlterTableOperationType getType(final byte value) { + public static AlterOrDropTableOperationType getType(final byte value) { switch (value) { case 0: return ADD_COLUMN; From d6b8927167abbb6edc2b142e962c40c23b10785e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 10:57:11 +0800 Subject: [PATCH 032/109] Refactor --- .../apache/iotdb/confignode/manager/ConfigManager.java | 4 ++-- .../org/apache/iotdb/confignode/manager/IManager.java | 4 ++-- .../iotdb/confignode/manager/ProcedureManager.java | 10 +++++----- .../service/thrift/ConfigNodeRPCServiceProcessor.java | 6 +++--- .../iotdb/db/protocol/client/ConfigNodeClient.java | 6 +++--- .../config/executor/ClusterConfigTaskExecutor.java | 6 +++--- .../src/main/thrift/confignode.thrift | 4 ++-- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index ee19a62edafa..64feb93d898f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -132,9 +132,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartReq; import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartResp; import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; -import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TClusterParameters; @@ -2567,7 +2567,7 @@ public TSStatus createTable(final ByteBuffer tableInfo) { } @Override - public TSStatus alterTable(final TAlterTableReq req) { + public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) { final TSStatus status = confirmLeader(); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { switch (AlterOrDropTableOperationType.getType(req.operationType)) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 640c82d6653f..f9accea5a822 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -60,9 +60,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartReq; import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartResp; import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; -import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterResp; @@ -832,7 +832,7 @@ TDataPartitionTableResp getOrCreateDataPartition( TSStatus createTable(final ByteBuffer tableInfo); - TSStatus alterTable(final TAlterTableReq req); + TSStatus alterOrDropTable(final TAlterOrDropTableReq req); TShowTableResp showTables(final String database); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index efb868327e86..77e456e7d148 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -111,8 +111,8 @@ import org.apache.iotdb.confignode.procedure.store.ProcedureFactory; import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq; import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq; import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq; @@ -1496,7 +1496,7 @@ public TSStatus createTable(final String database, final TsTable table) { new CreateTableProcedure(database, table)); } - public TSStatus alterTableAddColumn(final TAlterTableReq req) { + public TSStatus alterTableAddColumn(final TAlterOrDropTableReq req) { return executeWithoutDuplicate( req.database, null, @@ -1510,7 +1510,7 @@ public TSStatus alterTableAddColumn(final TAlterTableReq req) { TsTableColumnSchemaUtil.deserializeColumnSchemaList(req.updateInfo))); } - public TSStatus alterTableSetProperties(final TAlterTableReq req) { + public TSStatus alterTableSetProperties(final TAlterOrDropTableReq req) { return executeWithoutDuplicate( req.database, null, @@ -1521,7 +1521,7 @@ public TSStatus alterTableSetProperties(final TAlterTableReq req) { req.database, req.tableName, req.queryId, ReadWriteIOUtils.readMap(req.updateInfo))); } - public TSStatus alterTableRenameColumn(final TAlterTableReq req) { + public TSStatus alterTableRenameColumn(final TAlterOrDropTableReq req) { return executeWithoutDuplicate( req.database, null, @@ -1536,7 +1536,7 @@ public TSStatus alterTableRenameColumn(final TAlterTableReq req) { ReadWriteIOUtils.readString(req.updateInfo))); } - public TSStatus alterTableDropColumn(final TAlterTableReq req) { + public TSStatus alterTableDropColumn(final TAlterOrDropTableReq req) { return executeWithoutDuplicate( req.database, null, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index d64e82026a40..077bafcc0380 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -86,9 +86,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartResp; import org.apache.iotdb.confignode.rpc.thrift.TAddConsensusGroupReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; -import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; @@ -1299,8 +1299,8 @@ public TSStatus createTable(final ByteBuffer tableInfo) { } @Override - public TSStatus alterTable(final TAlterTableReq req) { - return configManager.alterTable(req); + public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) { + return configManager.alterOrDropTable(req); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index f3c6a07307d1..dc5b9c5b1729 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -49,9 +49,9 @@ import org.apache.iotdb.confignode.rpc.thrift.TAINodeRestartResp; import org.apache.iotdb.confignode.rpc.thrift.TAddConsensusGroupReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; -import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthizedPatternTreeResp; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerReq; import org.apache.iotdb.confignode.rpc.thrift.TAuthorizerResp; @@ -1269,9 +1269,9 @@ public TSStatus createTable(final ByteBuffer tableInfo) throws TException { } @Override - public TSStatus alterTable(final TAlterTableReq req) throws TException { + public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) throws TException { return executeRemoteCallWithRetry( - () -> client.alterTable(req), status -> !updateConfigNodeLeader(status)); + () -> client.alterOrDropTable(req), status -> !updateConfigNodeLeader(status)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 3d21c47a13c1..dcd5f37c4d95 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -61,9 +61,9 @@ import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TimePartitionUtils; import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq; +import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TAlterSchemaTemplateReq; -import org.apache.iotdb.confignode.rpc.thrift.TAlterTableReq; import org.apache.iotdb.confignode.rpc.thrift.TCountDatabaseResp; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListReq; import org.apache.iotdb.confignode.rpc.thrift.TCountTimeSlotListResp; @@ -3421,7 +3421,7 @@ private TSStatus sendAlterReq2ConfigNode( final ConfigNodeClient client) throws ClientManagerException, TException { TSStatus tsStatus; - final TAlterTableReq req = new TAlterTableReq(); + final TAlterOrDropTableReq req = new TAlterOrDropTableReq(); req.setDatabase(database); req.setTableName(tableName); req.setQueryId(queryId); @@ -3430,7 +3430,7 @@ private TSStatus sendAlterReq2ConfigNode( do { try { - tsStatus = client.alterTable(req); + tsStatus = client.alterOrDropTable(req); } catch (final TTransportException e) { if (e.getType() == TTransportException.TIMED_OUT || e.getCause() instanceof SocketTimeoutException) { diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 5c1ad65b4fbd..d7d8828d356f 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1031,7 +1031,7 @@ enum TTestOperation { // Table // ==================================================== -struct TAlterTableReq { +struct TAlterOrDropTableReq { 1: required string database 2: required string tableName 3: required string queryId @@ -1787,7 +1787,7 @@ service IConfigNodeRPCService { common.TSStatus createTable(binary tableInfo) - common.TSStatus alterTable(TAlterTableReq req) + common.TSStatus alterOrDropTable(TAlterOrDropTableReq req) TShowTableResp showTables(string database) From 987a89fa4cc84224061b424e4e576b0d5219faa9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:06:57 +0800 Subject: [PATCH 033/109] drop table proc --- .../confignode/manager/ConfigManager.java | 2 + .../confignode/manager/ProcedureManager.java | 11 +++ .../impl/schema/table/DropTableProcedure.java | 72 +++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 64feb93d898f..a88e4824e61e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -2579,6 +2579,8 @@ public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) { return procedureManager.alterTableRenameColumn(req); case DROP_COLUMN: return procedureManager.alterTableDropColumn(req); + case DROP_TABLE: + return procedureManager.dropTable(req); default: throw new IllegalArgumentException(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index 77e456e7d148..c9fe40ae1038 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -89,6 +89,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure; @@ -1547,6 +1548,16 @@ public TSStatus alterTableDropColumn(final TAlterOrDropTableReq req) { req.database, req.tableName, req.queryId, ReadWriteIOUtils.readString(req.updateInfo))); } + public TSStatus dropTable(final TAlterOrDropTableReq req) { + return executeWithoutDuplicate( + req.database, + null, + req.tableName, + req.queryId, + ProcedureType.DROP_TABLE_PROCEDURE, + new DropTableProcedure(req.database, req.tableName, req.queryId)); + } + private TSStatus executeWithoutDuplicate( final String database, final TsTable table, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java new file mode 100644 index 000000000000..67b6b458feef --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.confignode.procedure.exception.ProcedureException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; +import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; +import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; +import org.apache.iotdb.confignode.procedure.state.schema.CreateTableState; + +import java.io.IOException; + +public class DropTableProcedure + extends StateMachineProcedure { + + protected String database; + protected String tableName; + protected String queryId; + + public DropTableProcedure() {} + + public DropTableProcedure(final String database, final String tableName, final String queryId) { + this.database = database; + this.tableName = tableName; + this.queryId = queryId; + } + + @Override + protected Flow executeFromState( + final ConfigNodeProcedureEnv configNodeProcedureEnv, final CreateTableState state) + throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { + return null; + } + + @Override + protected void rollbackState( + final ConfigNodeProcedureEnv configNodeProcedureEnv, final CreateTableState state) + throws IOException, InterruptedException, ProcedureException {} + + @Override + protected CreateTableState getState(final int stateId) { + return null; + } + + @Override + protected int getStateId(final CreateTableState state) { + return 0; + } + + @Override + protected CreateTableState getInitialState() { + return null; + } +} From 1458de08c78c97d0bcd41f719e6ce50f35a45554 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:09:53 +0800 Subject: [PATCH 034/109] Create DropTableState.java --- .../state/schema/DropTableState.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java new file mode 100644 index 000000000000..d488fe079d28 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.state.schema; + +public enum DropTableState { + CHECK_TABLE_EXISTENCE, + INVALIDATE_CACHE, + DELETE_DATA, + DELETE_SCHEMA, + DROP_TABLE, +} From 33ece4f5cb13b503e248bd7383ec4cf17724bc86 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:16:21 +0800 Subject: [PATCH 035/109] partial --- .../schema/table/CreateTableProcedure.java | 5 +++++ .../impl/schema/table/DropTableProcedure.java | 20 +++++++++---------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index fc6944eff42b..d2c73aa8d659 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -300,6 +300,11 @@ private void commitReleaseTable(final ConfigNodeProcedureEnv env) { } } + @Override + protected boolean isRollbackSupported(final CreateTableState state) { + return true; + } + @Override protected void rollbackState(final ConfigNodeProcedureEnv env, final CreateTableState state) throws IOException, InterruptedException, ProcedureException { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 67b6b458feef..3e76a992a3b8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -24,12 +24,12 @@ import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; -import org.apache.iotdb.confignode.procedure.state.schema.CreateTableState; +import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; import java.io.IOException; public class DropTableProcedure - extends StateMachineProcedure { + extends StateMachineProcedure { protected String database; protected String tableName; @@ -45,28 +45,28 @@ public DropTableProcedure(final String database, final String tableName, final S @Override protected Flow executeFromState( - final ConfigNodeProcedureEnv configNodeProcedureEnv, final CreateTableState state) + final ConfigNodeProcedureEnv configNodeProcedureEnv, final DropTableState dropTableState) throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { return null; } @Override protected void rollbackState( - final ConfigNodeProcedureEnv configNodeProcedureEnv, final CreateTableState state) + final ConfigNodeProcedureEnv configNodeProcedureEnv, final DropTableState dropTableState) throws IOException, InterruptedException, ProcedureException {} @Override - protected CreateTableState getState(final int stateId) { - return null; + protected DropTableState getState(final int stateId) { + return DropTableState.values()[stateId]; } @Override - protected int getStateId(final CreateTableState state) { - return 0; + protected int getStateId(final DropTableState dropTableState) { + return dropTableState.ordinal(); } @Override - protected CreateTableState getInitialState() { - return null; + protected DropTableState getInitialState() { + return DropTableState.CHECK_TABLE_EXISTENCE; } } From ab4f51c488be969ec8436f9e7e12b98479b5391c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:17:32 +0800 Subject: [PATCH 036/109] Update AbstractAlterTableProcedure.java --- .../impl/schema/table/AbstractAlterTableProcedure.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java index 85bb3f33c5ec..d5cf6a62128f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java @@ -99,6 +99,11 @@ protected void commitRelease(final ConfigNodeProcedureEnv env) { } } + @Override + protected final boolean isRollbackSupported(final T state) { + return true; + } + protected void rollbackPreRelease(final ConfigNodeProcedureEnv env) { final Map failedResults = SchemaUtils.rollbackPreRelease(database, table.getTableName(), env.getConfigManager()); From 8b87dfcf5592b6917381a26ff74eef590df2e500 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:20:07 +0800 Subject: [PATCH 037/109] Update DropTableProcedure.java --- .../impl/schema/table/DropTableProcedure.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 3e76a992a3b8..692bd6f821bb 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -26,7 +26,12 @@ import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; public class DropTableProcedure extends StateMachineProcedure { @@ -69,4 +74,40 @@ protected int getStateId(final DropTableState dropTableState) { protected DropTableState getInitialState() { return DropTableState.CHECK_TABLE_EXISTENCE; } + + @Override + public void serialize(final DataOutputStream stream) throws IOException { + super.serialize(stream); + + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + ReadWriteIOUtils.write(queryId, stream); + } + + @Override + public void deserialize(final ByteBuffer byteBuffer) { + super.deserialize(byteBuffer); + this.database = ReadWriteIOUtils.readString(byteBuffer); + this.tableName = ReadWriteIOUtils.readString(byteBuffer); + this.queryId = ReadWriteIOUtils.readString(byteBuffer); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DropTableProcedure that = (DropTableProcedure) o; + return Objects.equals(database, that.database) + && Objects.equals(tableName, that.tableName) + && Objects.equals(queryId, that.queryId); + } + + @Override + public int hashCode() { + return Objects.hash(database, tableName, queryId); + } } From 88e5e5468baad46975021b4680c8e821a2d2add1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:21:47 +0800 Subject: [PATCH 038/109] Update ProcedureFactory.java --- .../iotdb/confignode/procedure/store/ProcedureFactory.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java index 072add32f91d..9cf5011d5e41 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java @@ -53,6 +53,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure; import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.AlterConsumerGroupProcedure; @@ -205,6 +206,9 @@ public Procedure create(ByteBuffer buffer) throws IOException { case DROP_TABLE_COLUMN_PROCEDURE: procedure = new DropTableColumnProcedure(); break; + case DROP_TABLE_PROCEDURE: + procedure = new DropTableProcedure(); + break; case CREATE_PIPE_PLUGIN_PROCEDURE: procedure = new CreatePipePluginProcedure(); break; From 31f3caee12a3d034ef439c7d677e20685012dda7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 11:23:57 +0800 Subject: [PATCH 039/109] Create DropTableProcedureTest.java --- .../schema/table/DropTableProcedureTest.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedureTest.java diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedureTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedureTest.java new file mode 100644 index 000000000000..394cee014227 --- /dev/null +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedureTest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.procedure.impl.schema.table; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class DropTableProcedureTest { + @Test + public void serializeDeserializeTest() throws IllegalPathException, IOException { + final DropTableProcedure dropTableProcedure = + new DropTableProcedure("root.database1", "table1", "0"); + + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); + dropTableProcedure.serialize(dataOutputStream); + + final ByteBuffer byteBuffer = ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); + + Assert.assertEquals(ProcedureType.DROP_TABLE_PROCEDURE.getTypeCode(), byteBuffer.getShort()); + + final DropTableProcedure deserializedProcedure = new DropTableProcedure(); + deserializedProcedure.deserialize(byteBuffer); + + Assert.assertEquals(dropTableProcedure, deserializedProcedure); + } +} From ec19ef1599143e670d21ea03e824af736b468d37 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:13:48 +0800 Subject: [PATCH 040/109] Refactor --- .../confignode/manager/ProcedureManager.java | 7 ++- ...=> AbstractAlterOrDropTableProcedure.java} | 11 ++-- .../schema/table/AddTableColumnProcedure.java | 3 +- .../table/DropTableColumnProcedure.java | 3 +- .../impl/schema/table/DropTableProcedure.java | 59 +++---------------- .../table/RenameTableColumnProcedure.java | 2 +- .../table/SetTablePropertiesProcedure.java | 2 +- 7 files changed, 24 insertions(+), 63 deletions(-) rename iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/{AbstractAlterTableProcedure.java => AbstractAlterOrDropTableProcedure.java} (93%) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java index c9fe40ae1038..93eb0f557226 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java @@ -85,7 +85,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.SetTTLProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure; -import org.apache.iotdb.confignode.procedure.impl.schema.table.AbstractAlterTableProcedure; +import org.apache.iotdb.confignode.procedure.impl.schema.table.AbstractAlterOrDropTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure; import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure; @@ -1615,8 +1615,9 @@ public Pair checkDuplicateTableTask( case SET_TABLE_PROPERTIES_PROCEDURE: case RENAME_TABLE_COLUMN_PROCEDURE: case DROP_TABLE_COLUMN_PROCEDURE: - final AbstractAlterTableProcedure alterTableProcedure = - (AbstractAlterTableProcedure) procedure; + case DROP_TABLE_PROCEDURE: + final AbstractAlterOrDropTableProcedure alterTableProcedure = + (AbstractAlterOrDropTableProcedure) procedure; if (type == thisType && queryId.equals(alterTableProcedure.getQueryId())) { return new Pair<>(procedure.getProcId(), false); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java similarity index 93% rename from iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java rename to iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java index d5cf6a62128f..7d5f2ef8f91e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java @@ -37,9 +37,10 @@ import java.util.Map; import java.util.Objects; -public abstract class AbstractAlterTableProcedure +public abstract class AbstractAlterOrDropTableProcedure extends StateMachineProcedure { - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractAlterTableProcedure.class); + private static final Logger LOGGER = + LoggerFactory.getLogger(AbstractAlterOrDropTableProcedure.class); protected String database; protected String tableName; @@ -47,9 +48,9 @@ public abstract class AbstractAlterTableProcedure protected TsTable table; - protected AbstractAlterTableProcedure() {} + protected AbstractAlterOrDropTableProcedure() {} - protected AbstractAlterTableProcedure( + protected AbstractAlterOrDropTableProcedure( final String database, final String tableName, final String queryId) { this.database = database; this.tableName = tableName; @@ -160,7 +161,7 @@ public boolean equals(final Object o) { if (o == null || getClass() != o.getClass()) { return false; } - final AbstractAlterTableProcedure that = (AbstractAlterTableProcedure) o; + final AbstractAlterOrDropTableProcedure that = (AbstractAlterOrDropTableProcedure) o; return Objects.equals(database, that.database) && Objects.equals(tableName, that.tableName) && Objects.equals(queryId, that.queryId); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index 30d17c44b5c6..f9fa4aa9fcc3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -41,7 +41,8 @@ import java.nio.ByteBuffer; import java.util.List; -public class AddTableColumnProcedure extends AbstractAlterTableProcedure { +public class AddTableColumnProcedure + extends AbstractAlterOrDropTableProcedure { private static final Logger LOGGER = LoggerFactory.getLogger(AddTableColumnProcedure.class); private List addedColumnList; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java index 4b2255c415ee..e029cb10bab2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java @@ -33,7 +33,8 @@ import java.nio.ByteBuffer; import java.util.Objects; -public class DropTableColumnProcedure extends AbstractAlterTableProcedure { +public class DropTableColumnProcedure + extends AbstractAlterOrDropTableProcedure { private String columnName; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 692bd6f821bb..181a5ca65ace 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -23,29 +23,22 @@ import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; -import org.apache.iotdb.confignode.procedure.impl.StateMachineProcedure; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Objects; - -public class DropTableProcedure - extends StateMachineProcedure { - protected String database; - protected String tableName; - protected String queryId; +public class DropTableProcedure extends AbstractAlterOrDropTableProcedure { public DropTableProcedure() {} public DropTableProcedure(final String database, final String tableName, final String queryId) { - this.database = database; - this.tableName = tableName; - this.queryId = queryId; + super(database, tableName, queryId); + } + + // Not used + @Override + protected String getActionMessage() { + return null; } @Override @@ -74,40 +67,4 @@ protected int getStateId(final DropTableState dropTableState) { protected DropTableState getInitialState() { return DropTableState.CHECK_TABLE_EXISTENCE; } - - @Override - public void serialize(final DataOutputStream stream) throws IOException { - super.serialize(stream); - - ReadWriteIOUtils.write(database, stream); - ReadWriteIOUtils.write(tableName, stream); - ReadWriteIOUtils.write(queryId, stream); - } - - @Override - public void deserialize(final ByteBuffer byteBuffer) { - super.deserialize(byteBuffer); - this.database = ReadWriteIOUtils.readString(byteBuffer); - this.tableName = ReadWriteIOUtils.readString(byteBuffer); - this.queryId = ReadWriteIOUtils.readString(byteBuffer); - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final DropTableProcedure that = (DropTableProcedure) o; - return Objects.equals(database, that.database) - && Objects.equals(tableName, that.tableName) - && Objects.equals(queryId, that.queryId); - } - - @Override - public int hashCode() { - return Objects.hash(database, tableName, queryId); - } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java index e5ad24c8a32b..4912271e7ad8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/RenameTableColumnProcedure.java @@ -41,7 +41,7 @@ import java.util.Objects; public class RenameTableColumnProcedure - extends AbstractAlterTableProcedure { + extends AbstractAlterOrDropTableProcedure { private static final Logger LOGGER = LoggerFactory.getLogger(RenameTableColumnProcedure.class); private String oldName; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java index 37ac5a71a884..1928505e288a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/SetTablePropertiesProcedure.java @@ -48,7 +48,7 @@ import static org.apache.iotdb.confignode.procedure.state.schema.SetTablePropertiesState.VALIDATE_TABLE; public class SetTablePropertiesProcedure - extends AbstractAlterTableProcedure { + extends AbstractAlterOrDropTableProcedure { private static final Logger LOGGER = LoggerFactory.getLogger(SetTablePropertiesProcedure.class); From e818309e3e752687c3b45014f79be516b9d84134 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 14:18:39 +0800 Subject: [PATCH 041/109] Update DropTableColumnState.java --- .../confignode/procedure/state/schema/DropTableColumnState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java index 1d0f88c0e9c2..331827c4b1c0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java @@ -19,6 +19,6 @@ package org.apache.iotdb.confignode.procedure.state.schema; -public class DropTableColumnState { +public enum DropTableColumnState { // TODO } From acb3642a9723b8e7bdbe11ef4855e4ad6fd15c4b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:19:34 +0800 Subject: [PATCH 042/109] Change --- .../AbstractAlterOrDropTableProcedure.java | 2 +- .../table/DropTableColumnProcedure.java | 2 +- .../impl/schema/table/DropTableProcedure.java | 57 +++++++++++++++++-- .../state/schema/DropTableColumnState.java | 2 +- .../state/schema/DropTableState.java | 4 +- 5 files changed, 58 insertions(+), 9 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java index 7d5f2ef8f91e..170148e7dcb8 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java @@ -101,7 +101,7 @@ protected void commitRelease(final ConfigNodeProcedureEnv env) { } @Override - protected final boolean isRollbackSupported(final T state) { + protected boolean isRollbackSupported(final T state) { return true; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java index e029cb10bab2..60c9386739af 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableColumnProcedure.java @@ -53,7 +53,7 @@ public DropTableColumnProcedure( @Override protected String getActionMessage() { - return null; + return "drop table column"; } @Override diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 181a5ca65ace..dcabd6a4206d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -25,10 +25,16 @@ import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.IOException; +import java.util.Objects; public class DropTableProcedure extends AbstractAlterOrDropTableProcedure { + private static final Logger LOGGER = LoggerFactory.getLogger(DropTableProcedure.class); + public DropTableProcedure() {} public DropTableProcedure(final String database, final String tableName, final String queryId) { @@ -43,15 +49,58 @@ protected String getActionMessage() { @Override protected Flow executeFromState( - final ConfigNodeProcedureEnv configNodeProcedureEnv, final DropTableState dropTableState) + final ConfigNodeProcedureEnv configNodeProcedureEnv, final DropTableState state) throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { - return null; + final long startTime = System.currentTimeMillis(); + try { + switch (state) { + case CHECK_AND_INVALIDATE_TABLE: + LOGGER.info("Check and invalidate table {}.{} when dropping table", database, tableName); + if (!isFailed() && Objects.isNull(table)) { + LOGGER.info( + "The updated table has the same properties with the original one. Skip the procedure."); + return Flow.NO_MORE_STATE; + } + break; + case INVALIDATE_CACHE: + LOGGER.info( + "Invalidating cache for table {}.{} when invalidating cache", database, tableName); + break; + case DELETE_DATA: + LOGGER.info("Deleting data for table {}.{}", database, tableName); + break; + case DELETE_DEVICES: + LOGGER.info("Deleting devices for table {}.{} when dropping table", database, tableName); + return Flow.NO_MORE_STATE; + case DROP_TABLE: + LOGGER.info("Dropping table {}.{} on configNode", database, tableName); + break; + default: + setFailure(new ProcedureException("Unrecognized DropTableState " + state)); + return Flow.NO_MORE_STATE; + } + return Flow.HAS_MORE_STATE; + } finally { + LOGGER.info( + "DropTable-{}.{}-{} costs {}ms", + database, + tableName, + state, + (System.currentTimeMillis() - startTime)); + } + } + + @Override + protected boolean isRollbackSupported(final DropTableState state) { + return false; } @Override protected void rollbackState( final ConfigNodeProcedureEnv configNodeProcedureEnv, final DropTableState dropTableState) - throws IOException, InterruptedException, ProcedureException {} + throws IOException, InterruptedException, ProcedureException { + // Do nothing + } @Override protected DropTableState getState(final int stateId) { @@ -65,6 +114,6 @@ protected int getStateId(final DropTableState dropTableState) { @Override protected DropTableState getInitialState() { - return DropTableState.CHECK_TABLE_EXISTENCE; + return DropTableState.CHECK_AND_INVALIDATE_TABLE; } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java index 331827c4b1c0..a9c22d1d439d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableColumnState.java @@ -20,5 +20,5 @@ package org.apache.iotdb.confignode.procedure.state.schema; public enum DropTableColumnState { - // TODO +// TODO } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java index d488fe079d28..1a4a7b222dc3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/state/schema/DropTableState.java @@ -20,9 +20,9 @@ package org.apache.iotdb.confignode.procedure.state.schema; public enum DropTableState { - CHECK_TABLE_EXISTENCE, + CHECK_AND_INVALIDATE_TABLE, INVALIDATE_CACHE, DELETE_DATA, - DELETE_SCHEMA, + DELETE_DEVICES, DROP_TABLE, } From dea1455171fb5a901ac57ac9928266e5a1d118f1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Mon, 14 Oct 2024 21:20:56 +0800 Subject: [PATCH 043/109] Refactor --- .../confignode/procedure/impl/StateMachineProcedure.java | 2 +- .../impl/schema/table/AbstractAlterOrDropTableProcedure.java | 4 +++- .../procedure/impl/schema/table/AddTableColumnProcedure.java | 4 +++- .../procedure/impl/schema/table/DropTableProcedure.java | 4 +++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/StateMachineProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/StateMachineProcedure.java index ed32ca6f0755..46529471017a 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/StateMachineProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/StateMachineProcedure.java @@ -74,7 +74,7 @@ protected StateMachineProcedure() { this(false); } - protected StateMachineProcedure(boolean isGeneratedByPipe) { + protected StateMachineProcedure(final boolean isGeneratedByPipe) { this.isGeneratedByPipe = isGeneratedByPipe; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java index 170148e7dcb8..7b8ee517e85b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AbstractAlterOrDropTableProcedure.java @@ -48,7 +48,9 @@ public abstract class AbstractAlterOrDropTableProcedure protected TsTable table; - protected AbstractAlterOrDropTableProcedure() {} + protected AbstractAlterOrDropTableProcedure() { + super(); + } protected AbstractAlterOrDropTableProcedure( final String database, final String tableName, final String queryId) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index f9fa4aa9fcc3..0f0c1a087b5d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -47,7 +47,9 @@ public class AddTableColumnProcedure private static final Logger LOGGER = LoggerFactory.getLogger(AddTableColumnProcedure.class); private List addedColumnList; - public AddTableColumnProcedure() {} + public AddTableColumnProcedure() { + super(); + } public AddTableColumnProcedure( final String database, diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index dcabd6a4206d..0f4fbab0f401 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -35,7 +35,9 @@ public class DropTableProcedure extends AbstractAlterOrDropTableProcedure Date: Mon, 14 Oct 2024 21:25:19 +0800 Subject: [PATCH 044/109] Update DropTableProcedure.java --- .../procedure/impl/schema/table/DropTableProcedure.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 0f4fbab0f401..ac9ddc740f6f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -92,6 +92,8 @@ protected Flow executeFromState( } } + private void checkAndInvalidateTable(final ConfigNodeProcedureEnv env) {} + @Override protected boolean isRollbackSupported(final DropTableState state) { return false; From c46c9fe62f728e6c9426650369c98067ec3a04e7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 15:15:59 +0800 Subject: [PATCH 045/109] Refactor --- .../consensus/request/ConfigPhysicalPlan.java | 4 ++ .../request/ConfigPhysicalPlanType.java | 1 + .../write/table/AbstractTablePlan.java | 69 +++++++++++++++++++ .../write/table/AddTableColumnPlan.java | 39 +++-------- .../write/table/CommitCreateTablePlan.java | 40 +---------- .../write/table/InvalidateTablePlan.java | 33 +++++++++ .../write/table/PreCreateTablePlan.java | 6 +- .../write/table/RenameTableColumnPlan.java | 27 ++------ .../write/table/RollbackCreateTablePlan.java | 40 +---------- .../write/table/SetTablePropertiesPlan.java | 27 ++------ 10 files changed, 134 insertions(+), 152 deletions(-) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AbstractTablePlan.java create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/InvalidateTablePlan.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index 3b70afd30532..f688e89c3785 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -96,6 +96,7 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.InvalidateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; @@ -347,6 +348,9 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept case RenameTableColumn: plan = new RenameTableColumnPlan(); break; + case InvalidateTable: + plan = new InvalidateTablePlan(); + break; case CreatePipeSinkV1: plan = new CreatePipeSinkPlanV1(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 5a6c9542a45f..2194aacf1a06 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -163,6 +163,7 @@ public enum ConfigPhysicalPlanType { ShowTable((short) 855), FetchTable((short) 856), RenameTableColumn((short) 857), + InvalidateTable((short) 858), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AbstractTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AbstractTablePlan.java new file mode 100644 index 000000000000..6ee66ca94d4a --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AbstractTablePlan.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +abstract class AbstractTablePlan extends ConfigPhysicalPlan { + + private String database; + + private String tableName; + + protected AbstractTablePlan(final ConfigPhysicalPlanType type) { + super(type); + } + + protected AbstractTablePlan( + final ConfigPhysicalPlanType type, final String database, final String tableName) { + super(type); + this.database = database; + this.tableName = tableName; + } + + public String getDatabase() { + return database; + } + + public String getTableName() { + return tableName; + } + + @Override + protected void serializeImpl(final DataOutputStream stream) throws IOException { + stream.writeShort(getType().getPlanType()); + + ReadWriteIOUtils.write(database, stream); + ReadWriteIOUtils.write(tableName, stream); + } + + @Override + protected void deserializeImpl(final ByteBuffer buffer) throws IOException { + this.database = ReadWriteIOUtils.readString(buffer); + this.tableName = ReadWriteIOUtils.readString(buffer); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AddTableColumnPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AddTableColumnPlan.java index eda7a26af74e..798ba0aeddde 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AddTableColumnPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/AddTableColumnPlan.java @@ -21,7 +21,6 @@ import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -31,11 +30,7 @@ import java.nio.ByteBuffer; import java.util.List; -public class AddTableColumnPlan extends ConfigPhysicalPlan { - - private String database; - - private String tableName; +public class AddTableColumnPlan extends AbstractTablePlan { private List columnSchemaList; @@ -46,25 +41,15 @@ public AddTableColumnPlan() { } public AddTableColumnPlan( - String database, - String tableName, - List columnSchemaList, - boolean isRollback) { - super(ConfigPhysicalPlanType.AddTableColumn); - this.database = database; - this.tableName = tableName; + final String database, + final String tableName, + final List columnSchemaList, + final boolean isRollback) { + super(ConfigPhysicalPlanType.AddTableColumn, database, tableName); this.columnSchemaList = columnSchemaList; this.isRollback = isRollback; } - public String getDatabase() { - return database; - } - - public String getTableName() { - return tableName; - } - public List getColumnSchemaList() { return columnSchemaList; } @@ -74,19 +59,15 @@ public boolean isRollback() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - - ReadWriteIOUtils.write(database, stream); - ReadWriteIOUtils.write(tableName, stream); + protected void serializeImpl(final DataOutputStream stream) throws IOException { + super.serializeImpl(stream); TsTableColumnSchemaUtil.serialize(columnSchemaList, stream); ReadWriteIOUtils.write(isRollback, stream); } @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.database = ReadWriteIOUtils.readString(buffer); - this.tableName = ReadWriteIOUtils.readString(buffer); + protected void deserializeImpl(final ByteBuffer buffer) throws IOException { + super.deserializeImpl(buffer); this.columnSchemaList = TsTableColumnSchemaUtil.deserializeColumnSchemaList(buffer); this.isRollback = ReadWriteIOUtils.readBool(buffer); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/CommitCreateTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/CommitCreateTablePlan.java index 8e723918f0c9..33516692fa23 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/CommitCreateTablePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/CommitCreateTablePlan.java @@ -19,49 +19,15 @@ package org.apache.iotdb.confignode.consensus.request.write.table; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class CommitCreateTablePlan extends ConfigPhysicalPlan { - - private String database; - - private String tableName; +public class CommitCreateTablePlan extends AbstractTablePlan { public CommitCreateTablePlan() { super(ConfigPhysicalPlanType.CommitCreateTable); } - public CommitCreateTablePlan(String database, String tableName) { - super(ConfigPhysicalPlanType.CommitCreateTable); - this.database = database; - this.tableName = tableName; - } - - public String getDatabase() { - return database; - } - - public String getTableName() { - return tableName; - } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - ReadWriteIOUtils.write(database, stream); - ReadWriteIOUtils.write(tableName, stream); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.database = ReadWriteIOUtils.readString(buffer); - this.tableName = ReadWriteIOUtils.readString(buffer); + public CommitCreateTablePlan(final String database, final String tableName) { + super(ConfigPhysicalPlanType.CommitCreateTable, database, tableName); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/InvalidateTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/InvalidateTablePlan.java new file mode 100644 index 000000000000..f7b16e5bda51 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/InvalidateTablePlan.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +public class InvalidateTablePlan extends AbstractTablePlan { + + public InvalidateTablePlan() { + super(ConfigPhysicalPlanType.InvalidateTable); + } + + public InvalidateTablePlan(final String database, final String tableName) { + super(ConfigPhysicalPlanType.InvalidateTable, database, tableName); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreCreateTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreCreateTablePlan.java index 13372161ee13..561f146b3996 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreCreateTablePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreCreateTablePlan.java @@ -39,7 +39,7 @@ public PreCreateTablePlan() { super(ConfigPhysicalPlanType.PreCreateTable); } - public PreCreateTablePlan(String database, TsTable table) { + public PreCreateTablePlan(final String database, final TsTable table) { super(ConfigPhysicalPlanType.PreCreateTable); this.database = database; this.table = table; @@ -54,14 +54,14 @@ public TsTable getTable() { } @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { + protected void serializeImpl(final DataOutputStream stream) throws IOException { stream.writeShort(getType().getPlanType()); ReadWriteIOUtils.write(database, stream); table.serialize(stream); } @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { + protected void deserializeImpl(final ByteBuffer buffer) throws IOException { database = ReadWriteIOUtils.readString(buffer); table = TsTable.deserialize(buffer); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RenameTableColumnPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RenameTableColumnPlan.java index 71cfc20e6c5c..134dceb076c0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RenameTableColumnPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RenameTableColumnPlan.java @@ -19,7 +19,6 @@ package org.apache.iotdb.confignode.consensus.request.write.table; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -28,11 +27,7 @@ import java.io.IOException; import java.nio.ByteBuffer; -public class RenameTableColumnPlan extends ConfigPhysicalPlan { - - private String database; - - private String tableName; +public class RenameTableColumnPlan extends AbstractTablePlan { private String oldName; private String newName; @@ -43,21 +38,11 @@ public RenameTableColumnPlan() { public RenameTableColumnPlan( final String database, final String tableName, final String oldName, final String newName) { - super(ConfigPhysicalPlanType.RenameTableColumn); - this.database = database; - this.tableName = tableName; + super(ConfigPhysicalPlanType.RenameTableColumn, database, tableName); this.oldName = oldName; this.newName = newName; } - public String getDatabase() { - return database; - } - - public String getTableName() { - return tableName; - } - public String getOldName() { return oldName; } @@ -68,18 +53,14 @@ public String getNewName() { @Override protected void serializeImpl(final DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - - ReadWriteIOUtils.write(database, stream); - ReadWriteIOUtils.write(tableName, stream); + super.serializeImpl(stream); ReadWriteIOUtils.write(oldName, stream); ReadWriteIOUtils.write(newName, stream); } @Override protected void deserializeImpl(final ByteBuffer buffer) throws IOException { - this.database = ReadWriteIOUtils.readString(buffer); - this.tableName = ReadWriteIOUtils.readString(buffer); + super.deserializeImpl(buffer); this.oldName = ReadWriteIOUtils.readString(buffer); this.newName = ReadWriteIOUtils.readString(buffer); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RollbackCreateTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RollbackCreateTablePlan.java index 5093817f11c4..11ea504648a4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RollbackCreateTablePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/RollbackCreateTablePlan.java @@ -19,49 +19,15 @@ package org.apache.iotdb.confignode.consensus.request.write.table; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; -import org.apache.tsfile.utils.ReadWriteIOUtils; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.nio.ByteBuffer; - -public class RollbackCreateTablePlan extends ConfigPhysicalPlan { - - private String database; - - private String tableName; +public class RollbackCreateTablePlan extends AbstractTablePlan { public RollbackCreateTablePlan() { super(ConfigPhysicalPlanType.RollbackCreateTable); } - public RollbackCreateTablePlan(String database, String tableName) { - super(ConfigPhysicalPlanType.RollbackCreateTable); - this.database = database; - this.tableName = tableName; - } - - public String getDatabase() { - return database; - } - - public String getTableName() { - return tableName; - } - - @Override - protected void serializeImpl(DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - ReadWriteIOUtils.write(database, stream); - ReadWriteIOUtils.write(tableName, stream); - } - - @Override - protected void deserializeImpl(ByteBuffer buffer) throws IOException { - this.database = ReadWriteIOUtils.readString(buffer); - this.tableName = ReadWriteIOUtils.readString(buffer); + public RollbackCreateTablePlan(final String database, final String tableName) { + super(ConfigPhysicalPlanType.RollbackCreateTable, database, tableName); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/SetTablePropertiesPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/SetTablePropertiesPlan.java index 88f3eb898717..8a2fa4642574 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/SetTablePropertiesPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/SetTablePropertiesPlan.java @@ -19,7 +19,6 @@ package org.apache.iotdb.confignode.consensus.request.write.table; -import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -29,11 +28,7 @@ import java.nio.ByteBuffer; import java.util.Map; -public class SetTablePropertiesPlan extends ConfigPhysicalPlan { - - private String database; - - private String tableName; +public class SetTablePropertiesPlan extends AbstractTablePlan { private Map properties; @@ -43,37 +38,23 @@ public SetTablePropertiesPlan() { public SetTablePropertiesPlan( final String database, final String tableName, final Map properties) { - super(ConfigPhysicalPlanType.SetTableProperties); - this.database = database; - this.tableName = tableName; + super(ConfigPhysicalPlanType.SetTableProperties, database, tableName); this.properties = properties; } - public String getDatabase() { - return database; - } - - public String getTableName() { - return tableName; - } - public Map getProperties() { return properties; } @Override protected void serializeImpl(final DataOutputStream stream) throws IOException { - stream.writeShort(getType().getPlanType()); - - ReadWriteIOUtils.write(database, stream); - ReadWriteIOUtils.write(tableName, stream); + super.serializeImpl(stream); ReadWriteIOUtils.write(properties, stream); } @Override protected void deserializeImpl(final ByteBuffer buffer) throws IOException { - this.database = ReadWriteIOUtils.readString(buffer); - this.tableName = ReadWriteIOUtils.readString(buffer); + super.deserializeImpl(buffer); this.properties = ReadWriteIOUtils.readMap(buffer); } } From da06323c614ce4bb75880217bab2b3b2a9eb089e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:24:38 +0800 Subject: [PATCH 046/109] Update DropTableProcedure.java --- .../impl/schema/table/DropTableProcedure.java | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index ac9ddc740f6f..40727c42986c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -19,17 +19,21 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table; +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.confignode.consensus.request.write.table.InvalidateTablePlan; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; +import org.apache.iotdb.consensus.exception.ConsensusException; +import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.Objects; public class DropTableProcedure extends AbstractAlterOrDropTableProcedure { @@ -50,19 +54,14 @@ protected String getActionMessage() { } @Override - protected Flow executeFromState( - final ConfigNodeProcedureEnv configNodeProcedureEnv, final DropTableState state) + protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DropTableState state) throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { final long startTime = System.currentTimeMillis(); try { switch (state) { case CHECK_AND_INVALIDATE_TABLE: LOGGER.info("Check and invalidate table {}.{} when dropping table", database, tableName); - if (!isFailed() && Objects.isNull(table)) { - LOGGER.info( - "The updated table has the same properties with the original one. Skip the procedure."); - return Flow.NO_MORE_STATE; - } + checkAndInvalidateTable(env); break; case INVALIDATE_CACHE: LOGGER.info( @@ -92,7 +91,22 @@ protected Flow executeFromState( } } - private void checkAndInvalidateTable(final ConfigNodeProcedureEnv env) {} + private void checkAndInvalidateTable(final ConfigNodeProcedureEnv env) { + final InvalidateTablePlan plan = new InvalidateTablePlan(database, tableName); + TSStatus status; + try { + status = env.getConfigManager().getConsensusManager().write(plan); + } catch (final ConsensusException e) { + LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); + status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + } + if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setNextState(DropTableState.INVALIDATE_CACHE); + } else { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + } @Override protected boolean isRollbackSupported(final DropTableState state) { From 1e48ced199243af47acd7cdeba7d4ae1b3918709 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:26:30 +0800 Subject: [PATCH 047/109] Update ConfigPhysicalPlanSerDeTest.java --- .../request/ConfigPhysicalPlanSerDeTest.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 3ae6d037dee1..08295082ec2a 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -120,6 +120,7 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.InvalidateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; @@ -1230,6 +1231,17 @@ public void SetTablePropertiesPlanTest() throws IOException { setTablePropertiesPlan0.getProperties(), setTablePropertiesPlan1.getProperties()); } + @Test + public void InvalidateTablePlanTest() throws IOException { + final InvalidateTablePlan invalidateTablePlan = + new InvalidateTablePlan("root.database1", "table1"); + final InvalidateTablePlan invalidateTablePlan1 = + (InvalidateTablePlan) + ConfigPhysicalPlan.Factory.create(invalidateTablePlan.serializeToByteBuffer()); + Assert.assertEquals(invalidateTablePlan.getDatabase(), invalidateTablePlan1.getDatabase()); + Assert.assertEquals(invalidateTablePlan.getTableName(), invalidateTablePlan1.getTableName()); + } + @Test public void AddTriggerInTablePlanTest() throws IOException, IllegalPathException { TriggerInformation triggerInformation = From 8cbda74125df2b4481069abd96f5fc1e566c038e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:33:43 +0800 Subject: [PATCH 048/109] Refactor --- .../consensus/request/ConfigPhysicalPlan.java | 6 +++--- .../request/ConfigPhysicalPlanType.java | 2 +- ...eTablePlan.java => PreDeleteTablePlan.java} | 10 +++++----- .../persistence/schema/ConfigMTree.java | 14 ++++++++++++++ .../impl/schema/table/DropTableProcedure.java | 8 ++++---- .../request/ConfigPhysicalPlanSerDeTest.java | 18 +++++++++--------- 6 files changed, 36 insertions(+), 22 deletions(-) rename iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/{InvalidateTablePlan.java => PreDeleteTablePlan.java} (76%) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index f688e89c3785..b41befbe08d0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -96,8 +96,8 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; -import org.apache.iotdb.confignode.consensus.request.write.table.InvalidateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.SetTablePropertiesPlan; @@ -348,8 +348,8 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept case RenameTableColumn: plan = new RenameTableColumnPlan(); break; - case InvalidateTable: - plan = new InvalidateTablePlan(); + case preDeleteTable: + plan = new PreDeleteTablePlan(); break; case CreatePipeSinkV1: plan = new CreatePipeSinkPlanV1(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 2194aacf1a06..6bd6a73c0d4f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -163,7 +163,7 @@ public enum ConfigPhysicalPlanType { ShowTable((short) 855), FetchTable((short) 856), RenameTableColumn((short) 857), - InvalidateTable((short) 858), + preDeleteTable((short) 858), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/InvalidateTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDeleteTablePlan.java similarity index 76% rename from iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/InvalidateTablePlan.java rename to iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDeleteTablePlan.java index f7b16e5bda51..180292e41a83 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/InvalidateTablePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDeleteTablePlan.java @@ -21,13 +21,13 @@ import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; -public class InvalidateTablePlan extends AbstractTablePlan { +public class PreDeleteTablePlan extends AbstractTablePlan { - public InvalidateTablePlan() { - super(ConfigPhysicalPlanType.InvalidateTable); + public PreDeleteTablePlan() { + super(ConfigPhysicalPlanType.preDeleteTable); } - public InvalidateTablePlan(final String database, final String tableName) { - super(ConfigPhysicalPlanType.InvalidateTable, database, tableName); + public PreDeleteTablePlan(final String database, final String tableName) { + super(ConfigPhysicalPlanType.preDeleteTable, database, tableName); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index 95b15bf4cdf2..15fc12b370ec 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -673,6 +673,20 @@ public void commitCreateTable(final PartialPath database, final String tableName tableNode.setStatus(TableNodeStatus.USING); } + public void preDeleteTable(final PartialPath database, final String tableName) + throws MetadataException { + final IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + if (!databaseNode.hasChild(tableName)) { + throw new TableNotExistsException( + database.getFullPath().substring(ROOT.length() + 1), tableName); + } + final ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); + if (!tableNode.getStatus().equals(TableNodeStatus.USING)) { + throw new IllegalStateException(); + } + tableNode.setStatus(TableNodeStatus.PRE_DELETE); + } + public List getAllUsingTablesUnderSpecificDatabase(final PartialPath databasePath) throws MetadataException { final IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(databasePath).getAsMNode(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 40727c42986c..835ec584b3c2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -21,7 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IoTDBException; -import org.apache.iotdb.confignode.consensus.request.write.table.InvalidateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; @@ -61,7 +61,7 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DropTabl switch (state) { case CHECK_AND_INVALIDATE_TABLE: LOGGER.info("Check and invalidate table {}.{} when dropping table", database, tableName); - checkAndInvalidateTable(env); + checkAndPreDeleteTable(env); break; case INVALIDATE_CACHE: LOGGER.info( @@ -91,8 +91,8 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DropTabl } } - private void checkAndInvalidateTable(final ConfigNodeProcedureEnv env) { - final InvalidateTablePlan plan = new InvalidateTablePlan(database, tableName); + private void checkAndPreDeleteTable(final ConfigNodeProcedureEnv env) { + final PreDeleteTablePlan plan = new PreDeleteTablePlan(database, tableName); TSStatus status; try { status = env.getConfigManager().getConsensusManager().write(plan); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 08295082ec2a..805a0e76cfa8 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -120,8 +120,8 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; -import org.apache.iotdb.confignode.consensus.request.write.table.InvalidateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.SetTablePropertiesPlan; @@ -1232,14 +1232,14 @@ public void SetTablePropertiesPlanTest() throws IOException { } @Test - public void InvalidateTablePlanTest() throws IOException { - final InvalidateTablePlan invalidateTablePlan = - new InvalidateTablePlan("root.database1", "table1"); - final InvalidateTablePlan invalidateTablePlan1 = - (InvalidateTablePlan) - ConfigPhysicalPlan.Factory.create(invalidateTablePlan.serializeToByteBuffer()); - Assert.assertEquals(invalidateTablePlan.getDatabase(), invalidateTablePlan1.getDatabase()); - Assert.assertEquals(invalidateTablePlan.getTableName(), invalidateTablePlan1.getTableName()); + public void PreDeleteTablePlanTest() throws IOException { + final PreDeleteTablePlan preDeleteTablePlan = + new PreDeleteTablePlan("root.database1", "table1"); + final PreDeleteTablePlan preDeleteTablePlan1 = + (PreDeleteTablePlan) + ConfigPhysicalPlan.Factory.create(preDeleteTablePlan.serializeToByteBuffer()); + Assert.assertEquals(preDeleteTablePlan.getDatabase(), preDeleteTablePlan1.getDatabase()); + Assert.assertEquals(preDeleteTablePlan.getTableName(), preDeleteTablePlan1.getTableName()); } @Test From bc2da99bc01e13706adfdf583979ebdfc96058a9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:38:49 +0800 Subject: [PATCH 049/109] Refactor --- .../consensus/request/ConfigPhysicalPlan.java | 2 +- .../consensus/request/ConfigPhysicalPlanType.java | 2 +- .../request/write/table/PreDeleteTablePlan.java | 4 ++-- .../persistence/executor/ConfigPlanExecutor.java | 3 +++ .../persistence/schema/ClusterSchemaInfo.java | 14 ++++++++++++++ 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index b41befbe08d0..73359fcf5d1d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -348,7 +348,7 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept case RenameTableColumn: plan = new RenameTableColumnPlan(); break; - case preDeleteTable: + case PreDeleteTable: plan = new PreDeleteTablePlan(); break; case CreatePipeSinkV1: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index 6bd6a73c0d4f..efd97bffb12e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -163,7 +163,7 @@ public enum ConfigPhysicalPlanType { ShowTable((short) 855), FetchTable((short) 856), RenameTableColumn((short) 857), - preDeleteTable((short) 858), + PreDeleteTable((short) 858), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDeleteTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDeleteTablePlan.java index 180292e41a83..a57ed7be418b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDeleteTablePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/PreDeleteTablePlan.java @@ -24,10 +24,10 @@ public class PreDeleteTablePlan extends AbstractTablePlan { public PreDeleteTablePlan() { - super(ConfigPhysicalPlanType.preDeleteTable); + super(ConfigPhysicalPlanType.PreDeleteTable); } public PreDeleteTablePlan(final String database, final String tableName) { - super(ConfigPhysicalPlanType.preDeleteTable, database, tableName); + super(ConfigPhysicalPlanType.PreDeleteTable, database, tableName); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index ef60f8647629..ca76da99a950 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -116,6 +116,7 @@ import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.SetTablePropertiesPlan; @@ -514,6 +515,8 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) return clusterSchemaInfo.renameTableColumn((RenameTableColumnPlan) physicalPlan); case SetTableProperties: return clusterSchemaInfo.setTableProperties((SetTablePropertiesPlan) physicalPlan); + case PreDeleteTable: + return clusterSchemaInfo.preDeleteTable((PreDeleteTablePlan) physicalPlan); case CreatePipeV2: return pipeInfo.createPipe((CreatePipePlanV2) physicalPlan); case SetPipeStatusV2: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 261cb00b10c0..07e66c23b0c5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -50,6 +50,7 @@ import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.RollbackCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.SetTablePropertiesPlan; @@ -1080,6 +1081,19 @@ public TSStatus commitCreateTable(final CommitCreateTablePlan plan) { } } + public TSStatus preDeleteTable(final PreDeleteTablePlan plan) { + databaseReadWriteLock.writeLock().lock(); + try { + mTree.preDeleteTable( + getQualifiedDatabasePartialPath(plan.getDatabase()), plan.getTableName()); + return RpcUtils.SUCCESS_STATUS; + } catch (final MetadataException e) { + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + public ShowTableResp showTables(final ShowTablePlan plan) { databaseReadWriteLock.readLock().lock(); try { From d34c7400a34aa3faa5aa2b9693cd2c05bec7ee6c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:40:34 +0800 Subject: [PATCH 050/109] Update ClusterSchemaInfo.java --- .../iotdb/confignode/persistence/schema/ClusterSchemaInfo.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 07e66c23b0c5..96d63d5da98c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -1085,7 +1085,7 @@ public TSStatus preDeleteTable(final PreDeleteTablePlan plan) { databaseReadWriteLock.writeLock().lock(); try { mTree.preDeleteTable( - getQualifiedDatabasePartialPath(plan.getDatabase()), plan.getTableName()); + getQualifiedDatabasePartialPath(plan.getDatabase()), plan.getTableName()); return RpcUtils.SUCCESS_STATUS; } catch (final MetadataException e) { return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); From 5f9a87fef72a2c73731da8357aa6dba2cc7e5000 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 17:51:01 +0800 Subject: [PATCH 051/109] DropTablePlan --- .../consensus/request/ConfigPhysicalPlan.java | 4 +++ .../request/ConfigPhysicalPlanType.java | 1 + .../request/write/table/DropTablePlan.java | 32 +++++++++++++++++++ .../executor/ConfigPlanExecutor.java | 3 ++ .../persistence/schema/ClusterSchemaInfo.java | 13 ++++++++ .../persistence/schema/ConfigMTree.java | 6 ++++ .../request/ConfigPhysicalPlanSerDeTest.java | 10 ++++++ 7 files changed, 69 insertions(+) create mode 100644 iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/DropTablePlan.java diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java index 73359fcf5d1d..d6366f1898ea 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java @@ -96,6 +96,7 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.DropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; @@ -351,6 +352,9 @@ public static ConfigPhysicalPlan create(final ByteBuffer buffer) throws IOExcept case PreDeleteTable: plan = new PreDeleteTablePlan(); break; + case DropTable: + plan = new DropTablePlan(); + break; case CreatePipeSinkV1: plan = new CreatePipeSinkPlanV1(); break; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index efd97bffb12e..68b28e7364a3 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@ -164,6 +164,7 @@ public enum ConfigPhysicalPlanType { FetchTable((short) 856), RenameTableColumn((short) 857), PreDeleteTable((short) 858), + DropTable((short) 859), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/DropTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/DropTablePlan.java new file mode 100644 index 000000000000..005f3cfe29e6 --- /dev/null +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/DropTablePlan.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.confignode.consensus.request.write.table; + +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType; + +public class DropTablePlan extends AbstractTablePlan { + public DropTablePlan() { + super(ConfigPhysicalPlanType.DropTable); + } + + public DropTablePlan(final String database, final String tableName) { + super(ConfigPhysicalPlanType.DropTable, database, tableName); + } +} diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java index ca76da99a950..f7f756679fc0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java @@ -115,6 +115,7 @@ import org.apache.iotdb.confignode.consensus.request.write.subscription.topic.runtime.TopicHandleMetaChangePlan; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.DropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; @@ -517,6 +518,8 @@ public TSStatus executeNonQueryPlan(ConfigPhysicalPlan physicalPlan) return clusterSchemaInfo.setTableProperties((SetTablePropertiesPlan) physicalPlan); case PreDeleteTable: return clusterSchemaInfo.preDeleteTable((PreDeleteTablePlan) physicalPlan); + case DropTable: + return clusterSchemaInfo.dropTable((DropTablePlan) physicalPlan); case CreatePipeV2: return pipeInfo.createPipe((CreatePipePlanV2) physicalPlan); case SetPipeStatusV2: diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 96d63d5da98c..bca28f93ef90 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -49,6 +49,7 @@ import org.apache.iotdb.confignode.consensus.request.write.database.SetTimePartitionIntervalPlan; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.DropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; @@ -1094,6 +1095,18 @@ public TSStatus preDeleteTable(final PreDeleteTablePlan plan) { } } + public TSStatus dropTable(final DropTablePlan plan) { + databaseReadWriteLock.writeLock().lock(); + try { + mTree.dropTable(getQualifiedDatabasePartialPath(plan.getDatabase()), plan.getTableName()); + return RpcUtils.SUCCESS_STATUS; + } catch (final MetadataException e) { + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } finally { + databaseReadWriteLock.writeLock().unlock(); + } + } + public ShowTableResp showTables(final ShowTablePlan plan) { databaseReadWriteLock.readLock().lock(); try { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index 15fc12b370ec..e504d7e8bfc5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -687,6 +687,12 @@ public void preDeleteTable(final PartialPath database, final String tableName) tableNode.setStatus(TableNodeStatus.PRE_DELETE); } + public void dropTable(final PartialPath database, final String tableName) + throws MetadataException { + final IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(database).getAsMNode(); + store.deleteChild(databaseNode, tableName); + } + public List getAllUsingTablesUnderSpecificDatabase(final PartialPath databasePath) throws MetadataException { final IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(databasePath).getAsMNode(); diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java index 805a0e76cfa8..42a2173dc06e 100644 --- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java +++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java @@ -120,6 +120,7 @@ import org.apache.iotdb.confignode.consensus.request.write.sync.SetPipeStatusPlanV1; import org.apache.iotdb.confignode.consensus.request.write.table.AddTableColumnPlan; import org.apache.iotdb.confignode.consensus.request.write.table.CommitCreateTablePlan; +import org.apache.iotdb.confignode.consensus.request.write.table.DropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreCreateTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.RenameTableColumnPlan; @@ -1242,6 +1243,15 @@ public void PreDeleteTablePlanTest() throws IOException { Assert.assertEquals(preDeleteTablePlan.getTableName(), preDeleteTablePlan1.getTableName()); } + @Test + public void DropTablePlanTest() throws IOException { + final DropTablePlan dropTablePlan = new DropTablePlan("root.database1", "table1"); + final DropTablePlan dropTablePlan1 = + (DropTablePlan) ConfigPhysicalPlan.Factory.create(dropTablePlan.serializeToByteBuffer()); + Assert.assertEquals(dropTablePlan.getDatabase(), dropTablePlan1.getDatabase()); + Assert.assertEquals(dropTablePlan.getTableName(), dropTablePlan1.getTableName()); + } + @Test public void AddTriggerInTablePlanTest() throws IOException, IllegalPathException { TriggerInformation triggerInformation = From 977fbfaa75b021d243b3a66c50be573940007bd8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:12:03 +0800 Subject: [PATCH 052/109] small change --- .../iotdb/db/schemaengine/schemaregion/ISchemaRegion.java | 2 +- .../main/java/org/apache/iotdb/commons/schema/node/IMNode.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index 57da8f5d1d6f..eea9efcfc300 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -324,7 +324,7 @@ void createOrUpdateTableDevice(final CreateOrUpdateTableDeviceNode createOrUpdat void updateTableDeviceAttribute(final TableDeviceAttributeUpdateNode updateNode) throws MetadataException; - void deleteTableDevice(String table) throws MetadataException; + void deleteTableDevice(final String table) throws MetadataException; // endregion diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java index 56fe4bdacd10..af158554ef9e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java @@ -52,7 +52,7 @@ public interface IMNode> extends ITreeNode { N addChild(N child); - N deleteChild(String name); + N deleteChild(final String name); IMNodeContainer getChildren(); From c2a98ed573ccaed2b9ad2b2256c9230becfb41c3 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:22:04 +0800 Subject: [PATCH 053/109] Refactor --- .../manager/consensus/ConsensusManager.java | 2 +- .../procedure/impl/schema/SchemaUtils.java | 17 +++++++++++++ .../schema/table/AddTableColumnProcedure.java | 1 - .../schema/table/CreateTableProcedure.java | 24 ++++--------------- .../impl/schema/table/DropTableProcedure.java | 22 +++++++++++++++++ .../apache/iotdb/db/utils/SchemaUtils.java | 1 + 6 files changed, 46 insertions(+), 21 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java index a339b4610470..56ae39563802 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/consensus/ConsensusManager.java @@ -324,7 +324,7 @@ public boolean removeConfigNodePeer(TConfigNodeLocation configNodeLocation) { * * @throws ConsensusException When write doesn't success */ - public TSStatus write(ConfigPhysicalPlan plan) throws ConsensusException { + public TSStatus write(final ConfigPhysicalPlan plan) throws ConsensusException { return consensusImpl.write(DEFAULT_CONSENSUS_GROUP_ID, plan); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java index 83bc5dcd8e53..fae64545ec70 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java @@ -32,7 +32,10 @@ import org.apache.iotdb.confignode.client.CnToDnRequestType; import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; +import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.manager.ConfigManager; +import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; +import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.schemaengine.template.Template; import org.apache.iotdb.mpp.rpc.thrift.TCheckSchemaRegionUsingTemplateReq; @@ -43,6 +46,7 @@ import org.apache.iotdb.rpc.TSStatusCode; import org.apache.tsfile.utils.ReadWriteIOUtils; +import org.slf4j.Logger; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; @@ -279,4 +283,17 @@ public static Map rollbackPreRelease( .filter(entry -> entry.getValue().getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } + + public static TSStatus executeInConsensusLayer( + final ConfigPhysicalPlan plan, final ConfigNodeProcedureEnv env, final Logger logger) { + TSStatus status; + try { + status = env.getConfigManager().getConsensusManager().write(plan); + } catch (final ConsensusException e) { + logger.warn("Failed in the read API executing the consensus layer due to: ", e); + status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + } + return status; + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java index 0f0c1a087b5d..5a710b001cc7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/AddTableColumnProcedure.java @@ -82,7 +82,6 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final AddTable LOGGER.info("Commit release info of table {}.{} when adding column", database, tableName); commitRelease(env); return Flow.NO_MORE_STATE; - default: setFailure(new ProcedureException("Unrecognized AddTableColumnState " + state)); return Flow.NO_MORE_STATE; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index d2c73aa8d659..fb074397faf0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -146,15 +146,8 @@ private void checkTableExistence(final ConfigNodeProcedureEnv env) { } private void preCreateTable(final ConfigNodeProcedureEnv env) { - final PreCreateTablePlan plan = new PreCreateTablePlan(database, table); - TSStatus status; - try { - status = env.getConfigManager().getConsensusManager().write(plan); - } catch (final ConsensusException e) { - LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); - status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); - status.setMessage(e.getMessage()); - } + final TSStatus status = + SchemaUtils.executeInConsensusLayer(new PreCreateTablePlan(database, table), env, LOGGER); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { setNextState(CreateTableState.PRE_RELEASE); } else { @@ -327,16 +320,9 @@ protected void rollbackState(final ConfigNodeProcedureEnv env, final CreateTable } private void rollbackCreate(final ConfigNodeProcedureEnv env) { - final RollbackCreateTablePlan plan = - new RollbackCreateTablePlan(database, table.getTableName()); - TSStatus status; - try { - status = env.getConfigManager().getConsensusManager().write(plan); - } catch (final ConsensusException e) { - LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); - status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); - status.setMessage(e.getMessage()); - } + final TSStatus status = + SchemaUtils.executeInConsensusLayer( + new RollbackCreateTablePlan(database, table.getTableName()), env, LOGGER); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn("Failed to rollback table creation {}.{}", database, table.getTableName()); setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 835ec584b3c2..050a39412c20 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.confignode.consensus.request.write.table.DropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; @@ -69,12 +70,14 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DropTabl break; case DELETE_DATA: LOGGER.info("Deleting data for table {}.{}", database, tableName); + deleteData(env); break; case DELETE_DEVICES: LOGGER.info("Deleting devices for table {}.{} when dropping table", database, tableName); return Flow.NO_MORE_STATE; case DROP_TABLE: LOGGER.info("Dropping table {}.{} on configNode", database, tableName); + dropTable(env); break; default: setFailure(new ProcedureException("Unrecognized DropTableState " + state)); @@ -108,6 +111,25 @@ private void checkAndPreDeleteTable(final ConfigNodeProcedureEnv env) { } } + private void deleteData(final ConfigNodeProcedureEnv env) { + // TODO + } + + private void dropTable(final ConfigNodeProcedureEnv env) { + final DropTablePlan plan = new DropTablePlan(database, tableName); + TSStatus status; + try { + status = env.getConfigManager().getConsensusManager().write(plan); + } catch (final ConsensusException e) { + LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); + status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); + status.setMessage(e.getMessage()); + } + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); + } + } + @Override protected boolean isRollbackSupported(final DropTableState state) { return false; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java index 29e2fcbc6bc7..773de36a067f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/SchemaUtils.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.utils; import org.apache.iotdb.common.rpc.thrift.TAggregationType; From c9cd3f42e185616d3ae9f09bc8172f6a2d2105d8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:27:07 +0800 Subject: [PATCH 054/109] Refactor --- .../procedure/impl/schema/SchemaUtils.java | 2 +- .../schema/table/CreateTableProcedure.java | 13 +++------- .../impl/schema/table/DropTableProcedure.java | 25 +++++-------------- 3 files changed, 10 insertions(+), 30 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java index fae64545ec70..d9e4315eff13 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java @@ -290,7 +290,7 @@ public static TSStatus executeInConsensusLayer( try { status = env.getConfigManager().getConsensusManager().write(plan); } catch (final ConsensusException e) { - logger.warn("Failed in the read API executing the consensus layer due to: ", e); + logger.warn("Failed in the write API executing the consensus layer due to: ", e); status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); status.setMessage(e.getMessage()); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index fb074397faf0..3f1e484eec72 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -41,7 +41,6 @@ import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.CreateTableState; import org.apache.iotdb.confignode.procedure.store.ProcedureType; -import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceReq; import org.apache.iotdb.mpp.rpc.thrift.TCheckTimeSeriesExistenceResp; import org.apache.iotdb.rpc.TSStatusCode; @@ -264,15 +263,9 @@ protected void onAllReplicasetFailure( } private void commitCreateTable(final ConfigNodeProcedureEnv env) { - final CommitCreateTablePlan plan = new CommitCreateTablePlan(database, table.getTableName()); - TSStatus status; - try { - status = env.getConfigManager().getConsensusManager().write(plan); - } catch (final ConsensusException e) { - LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); - status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); - status.setMessage(e.getMessage()); - } + final TSStatus status = + SchemaUtils.executeInConsensusLayer( + new CommitCreateTablePlan(database, table.getTableName()), env, LOGGER); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { setNextState(CreateTableState.COMMIT_RELEASE); } else { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 050a39412c20..495b8512eebe 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -27,8 +27,8 @@ import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; +import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; -import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; @@ -95,15 +95,9 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DropTabl } private void checkAndPreDeleteTable(final ConfigNodeProcedureEnv env) { - final PreDeleteTablePlan plan = new PreDeleteTablePlan(database, tableName); - TSStatus status; - try { - status = env.getConfigManager().getConsensusManager().write(plan); - } catch (final ConsensusException e) { - LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); - status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); - status.setMessage(e.getMessage()); - } + final TSStatus status = + SchemaUtils.executeInConsensusLayer( + new PreDeleteTablePlan(database, tableName), env, LOGGER); if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { setNextState(DropTableState.INVALIDATE_CACHE); } else { @@ -116,15 +110,8 @@ private void deleteData(final ConfigNodeProcedureEnv env) { } private void dropTable(final ConfigNodeProcedureEnv env) { - final DropTablePlan plan = new DropTablePlan(database, tableName); - TSStatus status; - try { - status = env.getConfigManager().getConsensusManager().write(plan); - } catch (final ConsensusException e) { - LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); - status = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); - status.setMessage(e.getMessage()); - } + final TSStatus status = + SchemaUtils.executeInConsensusLayer(new DropTablePlan(database, tableName), env, LOGGER); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { setFailure(new ProcedureException(new IoTDBException(status.getMessage(), status.getCode()))); } From 60117f2a2fe2f01b4df7a0d977a401d90918eb79 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:31:52 +0800 Subject: [PATCH 055/109] Update DeleteLogicalViewProcedure.java --- .../schema/DeleteLogicalViewProcedure.java | 154 +++++++++--------- 1 file changed, 80 insertions(+), 74 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java index 0a83770689c8..9bb7c094d8be 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java @@ -78,19 +78,20 @@ public class DeleteLogicalViewProcedure private static final String CONSENSUS_WRITE_ERROR = "Failed in the write API executing the consensus layer due to: "; - public DeleteLogicalViewProcedure(boolean isGeneratedByPipe) { + public DeleteLogicalViewProcedure(final boolean isGeneratedByPipe) { super(isGeneratedByPipe); } public DeleteLogicalViewProcedure( - String queryId, PathPatternTree patternTree, boolean isGeneratedByPipe) { + final String queryId, final PathPatternTree patternTree, final boolean isGeneratedByPipe) { super(isGeneratedByPipe); this.queryId = queryId; setPatternTree(patternTree); } @Override - protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteLogicalViewState state) + protected Flow executeFromState( + final ConfigNodeProcedureEnv env, final DeleteLogicalViewState state) throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException { long startTime = System.currentTimeMillis(); try { @@ -131,67 +132,67 @@ protected Flow executeFromState(ConfigNodeProcedureEnv env, DeleteLogicalViewSta // return the total num of timeseries in schemaengine black list private long constructBlackList(ConfigNodeProcedureEnv env) { - Map targetSchemaRegionGroup = + final Map targetSchemaRegionGroup = env.getConfigManager().getRelatedSchemaRegionGroup(patternTree); if (targetSchemaRegionGroup.isEmpty()) { return 0; } - List successResult = new ArrayList<>(); - DeleteLogicalViewRegionTaskExecutor constructBlackListTask = - new DeleteLogicalViewRegionTaskExecutor( - "construct view schema engine black list", - env, - targetSchemaRegionGroup, - CnToDnRequestType.CONSTRUCT_VIEW_SCHEMA_BLACK_LIST, - ((dataNodeLocation, consensusGroupIdList) -> - new TConstructViewSchemaBlackListReq(consensusGroupIdList, patternTreeBytes))) { - @Override - protected List processResponseOfOneDataNode( - TDataNodeLocation dataNodeLocation, - List consensusGroupIdList, - TSStatus response) { - List failedRegionList = new ArrayList<>(); - if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - successResult.add(response); - } else if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { - List subStatusList = response.getSubStatus(); - for (int i = 0; i < subStatusList.size(); i++) { - if (subStatusList.get(i).getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { - successResult.add(subStatusList.get(i)); + final List successResult = new ArrayList<>(); + final DeleteLogicalViewRegionTaskExecutor + constructBlackListTask = + new DeleteLogicalViewRegionTaskExecutor( + "construct view schema engine black list", + env, + targetSchemaRegionGroup, + CnToDnRequestType.CONSTRUCT_VIEW_SCHEMA_BLACK_LIST, + ((dataNodeLocation, consensusGroupIdList) -> + new TConstructViewSchemaBlackListReq(consensusGroupIdList, patternTreeBytes))) { + @Override + protected List processResponseOfOneDataNode( + TDataNodeLocation dataNodeLocation, + List consensusGroupIdList, + TSStatus response) { + final List failedRegionList = new ArrayList<>(); + if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + successResult.add(response); + } else if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { + List subStatusList = response.getSubStatus(); + for (int i = 0; i < subStatusList.size(); i++) { + if (subStatusList.get(i).getCode() + == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + successResult.add(subStatusList.get(i)); + } else { + failedRegionList.add(consensusGroupIdList.get(i)); + } + } } else { - failedRegionList.add(consensusGroupIdList.get(i)); + failedRegionList.addAll(consensusGroupIdList); } + return failedRegionList; } - } else { - failedRegionList.addAll(consensusGroupIdList); - } - return failedRegionList; - } - }; + }; constructBlackListTask.execute(); if (isFailed()) { return 0; } - long preDeletedNum = 0; - for (TSStatus resp : successResult) { - preDeletedNum += Long.parseLong(resp.getMessage()); - } - return preDeletedNum; + return successResult.stream() + .mapToLong(resp -> Long.parseLong(resp.getMessage())) + .reduce(0L, Long::sum); } - private void invalidateCache(ConfigNodeProcedureEnv env) { - Map dataNodeLocationMap = + private void invalidateCache(final ConfigNodeProcedureEnv env) { + final Map dataNodeLocationMap = env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); - DataNodeAsyncRequestContext clientHandler = + final DataNodeAsyncRequestContext clientHandler = new DataNodeAsyncRequestContext<>( CnToDnRequestType.INVALIDATE_MATCHED_SCHEMA_CACHE, new TInvalidateMatchedSchemaCacheReq(patternTreeBytes), dataNodeLocationMap); CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler); - Map statusMap = clientHandler.getResponseMap(); - for (TSStatus status : statusMap.values()) { + final Map statusMap = clientHandler.getResponseMap(); + for (final TSStatus status : statusMap.values()) { // all dataNodes must clear the related schemaengine cache if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.error("Failed to invalidate schemaengine cache of view {}", requestMessage); @@ -205,8 +206,8 @@ private void invalidateCache(ConfigNodeProcedureEnv env) { setNextState(DeleteLogicalViewState.DELETE_VIEW_SCHEMA); } - private void deleteViewSchema(ConfigNodeProcedureEnv env) { - DeleteLogicalViewRegionTaskExecutor deleteTimeSeriesTask = + private void deleteViewSchema(final ConfigNodeProcedureEnv env) { + final DeleteLogicalViewRegionTaskExecutor deleteTimeSeriesTask = new DeleteLogicalViewRegionTaskExecutor<>( "delete view in schema engine", env, @@ -218,7 +219,7 @@ private void deleteViewSchema(ConfigNodeProcedureEnv env) { deleteTimeSeriesTask.execute(); } - private void collectPayload4Pipe(ConfigNodeProcedureEnv env) { + private void collectPayload4Pipe(final ConfigNodeProcedureEnv env) { TSStatus result; try { result = @@ -228,7 +229,7 @@ private void collectPayload4Pipe(ConfigNodeProcedureEnv env) { isGeneratedByPipe ? new PipeEnrichedPlan(new PipeDeleteLogicalViewPlan(patternTreeBytes)) : new PipeDeleteLogicalViewPlan(patternTreeBytes)); - } catch (ConsensusException e) { + } catch (final ConsensusException e) { LOGGER.warn(CONSENSUS_WRITE_ERROR, e); result = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()); result.setMessage(e.getMessage()); @@ -240,7 +241,7 @@ private void collectPayload4Pipe(ConfigNodeProcedureEnv env) { @Override protected void rollbackState( - ConfigNodeProcedureEnv env, DeleteLogicalViewState deleteLogicalViewState) + final ConfigNodeProcedureEnv env, final DeleteLogicalViewState deleteLogicalViewState) throws IOException, InterruptedException, ProcedureException { DeleteLogicalViewRegionTaskExecutor rollbackStateTask = new DeleteLogicalViewRegionTaskExecutor<>( @@ -254,17 +255,17 @@ protected void rollbackState( } @Override - protected boolean isRollbackSupported(DeleteLogicalViewState deleteLogicalViewState) { + protected boolean isRollbackSupported(final DeleteLogicalViewState deleteLogicalViewState) { return true; } @Override - protected DeleteLogicalViewState getState(int stateId) { + protected DeleteLogicalViewState getState(final int stateId) { return DeleteLogicalViewState.values()[stateId]; } @Override - protected int getStateId(DeleteLogicalViewState deleteLogicalViewState) { + protected int getStateId(final DeleteLogicalViewState deleteLogicalViewState) { return deleteLogicalViewState.ordinal(); } @@ -281,25 +282,25 @@ public PathPatternTree getPatternTree() { return patternTree; } - public void setPatternTree(PathPatternTree patternTree) { + public void setPatternTree(final PathPatternTree patternTree) { this.patternTree = patternTree; requestMessage = patternTree.getAllPathPatterns().toString(); patternTreeBytes = preparePatternTreeBytesData(patternTree); } - private ByteBuffer preparePatternTreeBytesData(PathPatternTree patternTree) { - ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); + private ByteBuffer preparePatternTreeBytesData(final PathPatternTree patternTree) { + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); try { patternTree.serialize(dataOutputStream); - } catch (IOException ignored) { - + } catch (final IOException ignored) { + // DataOutputStream won't throw IOException } return ByteBuffer.wrap(byteArrayOutputStream.toByteArray()); } @Override - public void serialize(DataOutputStream stream) throws IOException { + public void serialize(final DataOutputStream stream) throws IOException { stream.writeShort( isGeneratedByPipe ? ProcedureType.PIPE_ENRICHED_DELETE_LOGICAL_VIEW_PROCEDURE.getTypeCode() @@ -310,17 +311,21 @@ public void serialize(DataOutputStream stream) throws IOException { } @Override - public void deserialize(ByteBuffer byteBuffer) { + public void deserialize(final ByteBuffer byteBuffer) { super.deserialize(byteBuffer); queryId = ReadWriteIOUtils.readString(byteBuffer); setPatternTree(PathPatternTree.deserialize(byteBuffer)); } @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - DeleteLogicalViewProcedure that = (DeleteLogicalViewProcedure) o; + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final DeleteLogicalViewProcedure that = (DeleteLogicalViewProcedure) o; return this.getProcId() == that.getProcId() && this.getCurrentState().equals(that.getCurrentState()) && this.getCycles() == that.getCycles() @@ -340,27 +345,27 @@ private class DeleteLogicalViewRegionTaskExecutor private final String taskName; DeleteLogicalViewRegionTaskExecutor( - String taskName, - ConfigNodeProcedureEnv env, - Map targetSchemaRegionGroup, - CnToDnRequestType dataNodeRequestType, - BiFunction, Q> dataNodeRequestGenerator) { + final String taskName, + final ConfigNodeProcedureEnv env, + final Map targetSchemaRegionGroup, + final CnToDnRequestType dataNodeRequestType, + final BiFunction, Q> dataNodeRequestGenerator) { super(env, targetSchemaRegionGroup, false, dataNodeRequestType, dataNodeRequestGenerator); this.taskName = taskName; } @Override protected List processResponseOfOneDataNode( - TDataNodeLocation dataNodeLocation, - List consensusGroupIdList, - TSStatus response) { - List failedRegionList = new ArrayList<>(); + final TDataNodeLocation dataNodeLocation, + final List consensusGroupIdList, + final TSStatus response) { + final List failedRegionList = new ArrayList<>(); if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { return failedRegionList; } if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { - List subStatus = response.getSubStatus(); + final List subStatus = response.getSubStatus(); for (int i = 0; i < subStatus.size(); i++) { if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { failedRegionList.add(consensusGroupIdList.get(i)); @@ -374,7 +379,8 @@ protected List processResponseOfOneDataNode( @Override protected void onAllReplicasetFailure( - TConsensusGroupId consensusGroupId, Set dataNodeLocationSet) { + final TConsensusGroupId consensusGroupId, + final Set dataNodeLocationSet) { setFailure( new ProcedureException( new MetadataException( From 78ea70e502823b6fefe1d4f8199f8e5518a1d7af Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 18:32:40 +0800 Subject: [PATCH 056/109] Update DeleteTimeSeriesProcedure.java --- .../impl/schema/DeleteTimeSeriesProcedure.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java index cd9133c0cf55..39143d99d1ef 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteTimeSeriesProcedure.java @@ -185,14 +185,12 @@ protected List processResponseOfOneDataNode( }; constructBlackListTask.execute(); - if (isFailed()) { - return 0; - } - - return successResult.stream() - .mapToLong(resp -> Long.parseLong(resp.getMessage())) - .reduce(Long::sum) - .orElse(0L); + return !isFailed() + ? successResult.stream() + .mapToLong(resp -> Long.parseLong(resp.getMessage())) + .reduce(Long::sum) + .orElse(0L) + : 0; } private void invalidateCache(final ConfigNodeProcedureEnv env) { From 3aab5f18f4241f4fe08eef79772dd34ed8a4f876 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Tue, 15 Oct 2024 20:17:46 +0800 Subject: [PATCH 057/109] Update DataNodeRegionTaskExecutor.java --- .../schema/DataNodeRegionTaskExecutor.java | 75 ++++++++++--------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DataNodeRegionTaskExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DataNodeRegionTaskExecutor.java index 8fd084a190ee..71e157d22be0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DataNodeRegionTaskExecutor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DataNodeRegionTaskExecutor.java @@ -52,11 +52,11 @@ public abstract class DataNodeRegionTaskExecutor { private boolean isInterrupted = false; protected DataNodeRegionTaskExecutor( - ConfigManager configManager, - Map targetRegionGroup, - boolean executeOnAllReplicaset, - CnToDnRequestType dataNodeRequestType, - BiFunction, Q> dataNodeRequestGenerator) { + final ConfigManager configManager, + final Map targetRegionGroup, + final boolean executeOnAllReplicaset, + final CnToDnRequestType dataNodeRequestType, + final BiFunction, Q> dataNodeRequestGenerator) { this.configManager = configManager; this.targetRegionGroup = targetRegionGroup; this.executeOnAllReplicaset = executeOnAllReplicaset; @@ -65,11 +65,11 @@ protected DataNodeRegionTaskExecutor( } protected DataNodeRegionTaskExecutor( - ConfigNodeProcedureEnv env, - Map targetRegionGroup, - boolean executeOnAllReplicaset, - CnToDnRequestType dataNodeRequestType, - BiFunction, Q> dataNodeRequestGenerator) { + final ConfigNodeProcedureEnv env, + final Map targetRegionGroup, + final boolean executeOnAllReplicaset, + final CnToDnRequestType dataNodeRequestType, + final BiFunction, Q> dataNodeRequestGenerator) { this.configManager = env.getConfigManager(); this.targetRegionGroup = targetRegionGroup; this.executeOnAllReplicaset = executeOnAllReplicaset; @@ -79,18 +79,18 @@ protected DataNodeRegionTaskExecutor( public void execute() { // organize region by dataNode - Map> failedHistory = new HashMap<>(); + final Map> failedHistory = new HashMap<>(); Map> dataNodeConsensusGroupIdMap = executeOnAllReplicaset ? getAllReplicaDataNodeRegionGroupMap(targetRegionGroup) : getLeaderDataNodeRegionGroupMap( configManager.getLoadManager().getRegionLeaderMap(), targetRegionGroup); while (!dataNodeConsensusGroupIdMap.isEmpty()) { - DataNodeAsyncRequestContext clientHandler = + final DataNodeAsyncRequestContext clientHandler = prepareRequestHandler(dataNodeConsensusGroupIdMap); CnToDnInternalServiceAsyncRequestManager.getInstance() .sendAsyncRequestWithRetry(clientHandler); - Map> currentFailedDataNodeMap = + final Map> currentFailedDataNodeMap = checkDataNodeExecutionResult(clientHandler.getResponseMap(), dataNodeConsensusGroupIdMap); if (isInterrupted) { @@ -115,10 +115,10 @@ public void execute() { } private DataNodeAsyncRequestContext prepareRequestHandler( - Map> dataNodeConsensusGroupIdMap) { - DataNodeAsyncRequestContext clientHandler = + final Map> dataNodeConsensusGroupIdMap) { + final DataNodeAsyncRequestContext clientHandler = new DataNodeAsyncRequestContext<>(dataNodeRequestType); - for (Map.Entry> entry : + for (final Map.Entry> entry : dataNodeConsensusGroupIdMap.entrySet()) { clientHandler.putNodeLocation(entry.getKey().getDataNodeId(), entry.getKey()); clientHandler.putRequest( @@ -129,13 +129,14 @@ private DataNodeAsyncRequestContext prepareRequestHandler( } private Map> checkDataNodeExecutionResult( - Map executionResult, - Map> dataNodeConsensusGroupIdMap) { - Map> currentFailedDataNodeMap = new HashMap<>(); - for (Map.Entry> entry : + final Map executionResult, + final Map> dataNodeConsensusGroupIdMap) { + final Map> currentFailedDataNodeMap = + new HashMap<>(); + for (final Map.Entry> entry : dataNodeConsensusGroupIdMap.entrySet()) { - R response = executionResult.get(entry.getKey().getDataNodeId()); - List failedRegionList = + final R response = executionResult.get(entry.getKey().getDataNodeId()); + final List failedRegionList = processResponseOfOneDataNode(entry.getKey(), entry.getValue(), response); if (failedRegionList.isEmpty()) { continue; @@ -146,8 +147,8 @@ private Map> checkDataNodeExecutionRe } private Map> getAvailableDataNodeLocationForRetry( - Map> failedDataNodeConsensusGroupIdMap, - Map> failedHistory) { + final Map> failedDataNodeConsensusGroupIdMap, + final Map> failedHistory) { failedDataNodeConsensusGroupIdMap.forEach( (k, v) -> { @@ -156,18 +157,21 @@ private Map> getAvailableDataNodeLoca } }); - Map> availableDataNodeLocation = new HashMap<>(); + final Map> availableDataNodeLocation = + new HashMap<>(); - Map leaderMap = configManager.getLoadManager().getRegionLeaderMap(); - for (List consensusGroupIdList : + final Map leaderMap = + configManager.getLoadManager().getRegionLeaderMap(); + for (final List consensusGroupIdList : failedDataNodeConsensusGroupIdMap.values()) { - for (TConsensusGroupId consensusGroupId : consensusGroupIdList) { - TRegionReplicaSet regionReplicaSet = targetRegionGroup.get(consensusGroupId); + for (final TConsensusGroupId consensusGroupId : consensusGroupIdList) { + final TRegionReplicaSet regionReplicaSet = targetRegionGroup.get(consensusGroupId); TDataNodeLocation selectedDataNode = null; - Integer leaderId = leaderMap.get(consensusGroupId); + final Integer leaderId = leaderMap.get(consensusGroupId); Set failedDataNodeSet; if (leaderId == null || leaderId == -1) { - for (TDataNodeLocation candidateDataNode : regionReplicaSet.getDataNodeLocations()) { + for (final TDataNodeLocation candidateDataNode : + regionReplicaSet.getDataNodeLocations()) { if ((failedDataNodeSet = failedHistory.get(consensusGroupId)) != null && failedDataNodeSet.contains(candidateDataNode)) { continue; @@ -177,7 +181,8 @@ private Map> getAvailableDataNodeLoca break; } } else { - for (TDataNodeLocation candidateDataNode : regionReplicaSet.getDataNodeLocations()) { + for (final TDataNodeLocation candidateDataNode : + regionReplicaSet.getDataNodeLocations()) { if ((failedDataNodeSet = failedHistory.get(consensusGroupId)) != null && failedDataNodeSet.contains(candidateDataNode)) { continue; @@ -214,12 +219,14 @@ protected final void interruptTask() { * with execution failure. */ protected abstract List processResponseOfOneDataNode( - TDataNodeLocation dataNodeLocation, List consensusGroupIdList, R response); + final TDataNodeLocation dataNodeLocation, + final List consensusGroupIdList, + final R response); /** * When all replicas failed on executing given task, the process defined by subclass will be * executed. */ protected abstract void onAllReplicasetFailure( - TConsensusGroupId consensusGroupId, Set dataNodeLocationSet); + final TConsensusGroupId consensusGroupId, final Set dataNodeLocationSet); } From fbc65f1d5c6946efe94349532d7264d6338a02b1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:26:28 +0800 Subject: [PATCH 058/109] Update DeleteLogicalViewProcedure.java --- .../impl/schema/DeleteLogicalViewProcedure.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java index 2faca54c3899..6b67da9aab9c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/DeleteLogicalViewProcedure.java @@ -160,11 +160,15 @@ protected List processResponseOfOneDataNode( if (subStatusList.get(i).getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { successResult.add(subStatusList.get(i)); } else { - failedRegionList.addAll(consensusGroupIdList); + failedRegionList.add(consensusGroupIdList.get(i)); } - return failedRegionList; } - }; + } else { + failedRegionList.addAll(consensusGroupIdList); + } + return failedRegionList; + } + }; constructBlackListTask.execute(); if (isFailed()) { From 3d3cda78deadd9149be29cfd20c9524dc90944ee Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:38:53 +0800 Subject: [PATCH 059/109] partial --- .../client/async/CnToDnAsyncRequestType.java | 2 + .../schema/table/CreateTableProcedure.java | 4 + .../impl/schema/table/DropTableProcedure.java | 110 +++++++++++++++++- 3 files changed, 115 insertions(+), 1 deletion(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java index fdac18e91aa3..bc04ab1cb4dc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java @@ -109,4 +109,6 @@ public enum CnToDnAsyncRequestType { // Table UPDATE_TABLE, + DELETE_DATA_FOR_DROP_TABLE, + DELETE_DEVICES_FOR_DROP_TABLE, } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index ffe2813adb42..b48072996eb1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -190,6 +190,10 @@ private void validateTimeSeriesExistence(final ConfigNodeProcedureEnv env) { final Map relatedSchemaRegionGroup = env.getConfigManager().getRelatedSchemaRegionGroup(patternTree); + if (relatedSchemaRegionGroup.isEmpty()) { + return; + } + final List respList = new ArrayList<>(); DataNodeRegionTaskExecutor regionTask = diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 495b8512eebe..5f9eb7906e78 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -19,14 +19,22 @@ package org.apache.iotdb.confignode.procedure.impl.schema.table; +import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId; +import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation; +import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet; import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.exception.IoTDBException; +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; import org.apache.iotdb.confignode.consensus.request.write.table.DropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; import org.apache.iotdb.confignode.procedure.exception.ProcedureException; import org.apache.iotdb.confignode.procedure.exception.ProcedureSuspendedException; import org.apache.iotdb.confignode.procedure.exception.ProcedureYieldException; +import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeRegionTaskExecutor; import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; import org.apache.iotdb.rpc.TSStatusCode; @@ -34,7 +42,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiFunction; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD; +import static org.apache.iotdb.commons.schema.SchemaConstant.ROOT; public class DropTableProcedure extends AbstractAlterOrDropTableProcedure { @@ -106,7 +124,25 @@ private void checkAndPreDeleteTable(final ConfigNodeProcedureEnv env) { } private void deleteData(final ConfigNodeProcedureEnv env) { - // TODO + final PathPatternTree patternTree = new PathPatternTree(); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); + final PartialPath path; + try { + path = new PartialPath(new String[] {ROOT, database, table.getTableName()}); + patternTree.appendPathPattern(path); + patternTree.appendPathPattern(path.concatAsMeasurementPath(MULTI_LEVEL_PATH_WILDCARD)); + patternTree.serialize(dataOutputStream); + } catch (final IOException e) { + LOGGER.warn("failed to serialize request for table {}.{}", database, table.getTableName(), e); + } + + final Map relatedDataRegionGroup = + env.getConfigManager().getRelatedDataRegionGroup(patternTree); + + if (relatedDataRegionGroup.isEmpty()) { + return; + } } private void dropTable(final ConfigNodeProcedureEnv env) { @@ -143,4 +179,76 @@ protected int getStateId(final DropTableState dropTableState) { protected DropTableState getInitialState() { return DropTableState.CHECK_AND_INVALIDATE_TABLE; } + + private class DropTableRegionTaskExecutor extends DataNodeRegionTaskExecutor { + + private final String taskName; + + DropTableRegionTaskExecutor( + final String taskName, + final ConfigNodeProcedureEnv env, + final Map targetSchemaRegionGroup, + final CnToDnAsyncRequestType dataNodeRequestType, + final BiFunction, Q> dataNodeRequestGenerator) { + super(env, targetSchemaRegionGroup, false, dataNodeRequestType, dataNodeRequestGenerator); + this.taskName = taskName; + } + + DropTableRegionTaskExecutor( + final String taskName, + final ConfigNodeProcedureEnv env, + final Map targetDataRegionGroup, + final boolean executeOnAllReplicaset, + final CnToDnAsyncRequestType dataNodeRequestType, + final BiFunction, Q> dataNodeRequestGenerator) { + super( + env, + targetDataRegionGroup, + executeOnAllReplicaset, + dataNodeRequestType, + dataNodeRequestGenerator); + this.taskName = taskName; + } + + @Override + protected List processResponseOfOneDataNode( + final TDataNodeLocation dataNodeLocation, + final List consensusGroupIdList, + final TSStatus response) { + final List failedRegionList = new ArrayList<>(); + if (response.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + return failedRegionList; + } + + if (response.getCode() == TSStatusCode.MULTIPLE_ERROR.getStatusCode()) { + final List subStatus = response.getSubStatus(); + for (int i = 0; i < subStatus.size(); i++) { + if (subStatus.get(i).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + failedRegionList.add(consensusGroupIdList.get(i)); + } + } + } else { + failedRegionList.addAll(consensusGroupIdList); + } + return failedRegionList; + } + + @Override + protected void onAllReplicasetFailure( + final TConsensusGroupId consensusGroupId, + final Set dataNodeLocationSet) { + setFailure( + new ProcedureException( + new MetadataException( + String.format( + "Drop table %s.%s failed when [%s] because failed to execute in all replicaset of %s %s. Failure nodes: %s", + database, + tableName, + taskName, + consensusGroupId.type, + consensusGroupId.id, + dataNodeLocationSet)))); + interruptTask(); + } + } } From b5d616d1963a68d2f53cee8311ceb7a049be1721 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:18:07 +0800 Subject: [PATCH 060/109] Async delete data --- ...oDnInternalServiceAsyncRequestManager.java | 6 ++++++ .../impl/schema/table/DropTableProcedure.java | 11 ++++++++++ .../impl/DataNodeInternalRPCServiceImpl.java | 21 ++++++++++++++----- .../src/main/thrift/datanode.thrift | 11 ++++++++++ 4 files changed, 44 insertions(+), 5 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java index 6e6b64bc3902..834e16f796a5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java @@ -62,6 +62,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TCreateTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TDeactivateTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDeleteSchemaReq; +import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDropTableReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteTimeSeriesReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteViewSchemaReq; import org.apache.iotdb.mpp.rpc.thrift.TDropFunctionInstanceReq; @@ -368,6 +369,11 @@ protected void initActionMapBuilder() { CnToDnAsyncRequestType.UPDATE_TABLE, (req, client, handler) -> client.updateTable((TUpdateTableReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.DELETE_DATA_FOR_DROP_TABLE, + (req, client, handler) -> + client.deleteDataForDropTable( + (TDeleteDataForDropTableReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( CnToDnAsyncRequestType.CLEAN_DATA_NODE_CACHE, (req, client, handler) -> diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 5f9eb7906e78..2b847bd70287 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -37,6 +37,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeRegionTaskExecutor; import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; +import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDropTableReq; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; @@ -143,6 +144,16 @@ private void deleteData(final ConfigNodeProcedureEnv env) { if (relatedDataRegionGroup.isEmpty()) { return; } + + new DropTableRegionTaskExecutor<>( + "delete data", + env, + relatedDataRegionGroup, + true, + CnToDnAsyncRequestType.DELETE_DATA_FOR_DROP_TABLE, + ((dataNodeLocation, consensusGroupIdList) -> + new TDeleteDataForDropTableReq( + new ArrayList<>(consensusGroupIdList), database, tableName))); } private void dropTable(final ConfigNodeProcedureEnv env) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 98be632a2e41..1c67241e0b2c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -68,6 +68,7 @@ import org.apache.iotdb.commons.trigger.TriggerInformation; import org.apache.iotdb.commons.udf.UDFInformation; import org.apache.iotdb.commons.udf.service.UDFManagementService; +import org.apache.iotdb.commons.utils.StatusUtils; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.exception.ConsensusException; @@ -191,6 +192,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp; import org.apache.iotdb.mpp.rpc.thrift.TDeactivateTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDeleteSchemaReq; +import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDropTableReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteTimeSeriesReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteViewSchemaReq; import org.apache.iotdb.mpp.rpc.thrift.TDropFunctionInstanceReq; @@ -663,8 +665,8 @@ public TSStatus deleteDataForDeleteSchema(TDeleteDataForDeleteSchemaReq req) { } @Override - public TSStatus deleteTimeSeries(TDeleteTimeSeriesReq req) throws TException { - PathPatternTree patternTree = + public TSStatus deleteTimeSeries(final TDeleteTimeSeriesReq req) throws TException { + final PathPatternTree patternTree = PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree())); return executeInternalSchemaTask( req.getSchemaRegionIdList(), @@ -1479,7 +1481,7 @@ public TFetchFragmentInstanceStatisticsResp fetchFragmentInstanceStatistics( } @Override - public TSStatus updateTable(TUpdateTableReq req) { + public TSStatus updateTable(final TUpdateTableReq req) { switch (TsTableInternalRPCType.getType(req.type)) { case PRE_CREATE_OR_ADD_COLUMN: DataNodeSchemaLockManager.getInstance().takeWriteLock(SchemaLockType.TIMESERIES_VS_TABLE); @@ -1511,8 +1513,17 @@ public TSStatus updateTable(TUpdateTableReq req) { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } - public TTestConnectionResp submitTestConnectionTask(TNodeLocations nodeLocations) - throws TException { + @Override + public TSStatus deleteDataForDropTable(final TDeleteDataForDropTableReq req) { + return executeInternalSchemaTask( + req.getDataRegionIdList(), + consensusGroupId -> { + // TODO + return StatusUtils.OK; + }); + } + + public TTestConnectionResp submitTestConnectionTask(TNodeLocations nodeLocations) { return new TTestConnectionResp( new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()), Stream.of( diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index e2fb51834dcc..9100b078fbd9 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -330,6 +330,12 @@ struct TUpdateTableReq { 2: required binary tableInfo } +struct TDeleteDataForDropTableReq { + 1: required list dataRegionIdList + 2: required string database + 3: required string tableName +} + struct TTsFilePieceReq { 1: required binary body 2: required string uuid @@ -1038,6 +1044,11 @@ service IDataNodeRPCService { */ common.TSStatus updateTable(TUpdateTableReq req) + /** + * Delete data for delete table + */ + common.TSStatus deleteDataForDropTable(TDeleteDataForDropTableReq req) + common.TTestConnectionResp submitTestConnectionTask(common.TNodeLocations nodeLocations) /** Empty rpc, only for connection test */ From 8396dbb26b75b8c51c4ca922f1a62751a9c7f3e5 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 11:31:39 +0800 Subject: [PATCH 061/109] Added tree size --- .../mnode/impl/ConfigBasicInternalMNode.java | 6 ++++++ .../impl/mem/mnode/impl/BasicInternalMNode.java | 5 +++++ .../mnode/impl/CachedBasicInternalMNode.java | 5 +++++ .../apache/iotdb/commons/schema/node/IMNode.java | 5 +++++ .../commons/schema/node/role/IInternalMNode.java | 16 ++++++++++++++++ 5 files changed, 37 insertions(+) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigBasicInternalMNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigBasicInternalMNode.java index b61d5570a608..eb99e1c8b52c 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigBasicInternalMNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigBasicInternalMNode.java @@ -19,6 +19,7 @@ package org.apache.iotdb.confignode.persistence.schema.mnode.impl; import org.apache.iotdb.commons.schema.node.MNodeType; +import org.apache.iotdb.commons.schema.node.role.IInternalMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeContainer; import org.apache.iotdb.confignode.persistence.schema.mnode.IConfigMNode; import org.apache.iotdb.confignode.persistence.schema.mnode.basic.ConfigBasicMNode; @@ -144,6 +145,11 @@ public int estimateSize() { return 8 + 80 + super.estimateSize(); } + @Override + public int estimateTreeSize() { + return estimateSize() + IInternalMNode.getChildrenTreeSize(children); + } + @Override public MNodeType getMNodeType() { return MNodeType.SG_INTERNAL; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/BasicInternalMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/BasicInternalMNode.java index 904a0a5b5ba1..15801ef35618 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/BasicInternalMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/BasicInternalMNode.java @@ -159,6 +159,11 @@ public int estimateSize() { return 8 + 80 + super.estimateSize() + 8 + (deviceInfo == null ? 0 : deviceInfo.estimateSize()); } + @Override + public int estimateTreeSize() { + return estimateSize() + IInternalMNode.getChildrenTreeSize(children); + } + @Override public MNodeType getMNodeType() { return deviceInfo == null ? MNodeType.INTERNAL : MNodeType.DEVICE; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedBasicInternalMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedBasicInternalMNode.java index 1b4e67e623df..7b29fd8d0c64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedBasicInternalMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedBasicInternalMNode.java @@ -176,6 +176,11 @@ public int estimateSize() { + (deviceInfo == null ? 0 : deviceInfo.estimateSize()); } + @Override + public int estimateTreeSize() { + return estimateSize() + IInternalMNode.getChildrenTreeSize(children); + } + @Override public boolean isDevice() { return getDeviceInfo() != null; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java index af158554ef9e..87ea5b00407a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.commons.schema.node; import org.apache.iotdb.commons.path.PartialPath; @@ -80,5 +81,9 @@ public interface IMNode> extends ITreeNode { int estimateSize(); + default int estimateTreeSize() { + return estimateSize(); + } + N getAsMNode(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/role/IInternalMNode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/role/IInternalMNode.java index ed69923b86d0..3cdfdfd5231a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/role/IInternalMNode.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/role/IInternalMNode.java @@ -22,6 +22,11 @@ import org.apache.iotdb.commons.schema.node.IMNode; import org.apache.iotdb.commons.schema.node.common.DeviceMNodeWrapper; import org.apache.iotdb.commons.schema.node.info.IDeviceInfo; +import org.apache.iotdb.commons.schema.node.utils.IMNodeContainer; + +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.Objects; public interface IInternalMNode> extends IMNode { @@ -52,4 +57,15 @@ default IDeviceMNode getAsDeviceMNode() { throw new UnsupportedOperationException("Wrong node type"); } } + + static int getChildrenTreeSize(final IMNodeContainer> children) { + return Objects.nonNull(children) + ? children.entrySet().stream() + .map( + entry -> + (int) RamUsageEstimator.sizeOf(entry.getKey()) + + entry.getValue().estimateTreeSize()) + .reduce(0, Integer::sum) + : 0; + } } From 793696184670d7b60cee2d215cbb409ee14a3e8d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 11:40:02 +0800 Subject: [PATCH 062/109] Change --- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 8 ++++++-- .../schemaregion/mtree/impl/mem/MemMTreeStore.java | 6 +++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 259449710bee..25ea90d1b584 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -1584,8 +1584,12 @@ protected void updateEntity(final IDeviceMNode node) { public void renameTableAttribute() {} - public void deleteTableDevice(final String tableName) { - storageGroupMNode.deleteChild(tableName); + public boolean deleteTableDevice(final String tableName) { + if (store.hasChild(storageGroupMNode, tableName)) { + store.deleteChild(storageGroupMNode, tableName); + return true; + } + return false; } // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java index 544a0534c9fa..21490bdf3896 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java @@ -123,7 +123,7 @@ public IMNodeIterator getTraverserIterator( } @Override - public IMemMNode addChild(IMemMNode parent, String childName, IMemMNode child) { + public IMemMNode addChild(final IMemMNode parent, final String childName, final IMemMNode child) { IMemMNode result = parent.addChild(childName, child); if (result == child) { requestMemory(child.estimateSize()); @@ -132,8 +132,8 @@ public IMemMNode addChild(IMemMNode parent, String childName, IMemMNode child) { } @Override - public void deleteChild(IMemMNode parent, String childName) { - releaseMemory(parent.deleteChild(childName).estimateSize()); + public void deleteChild(final IMemMNode parent, final String childName) { + releaseMemory(parent.deleteChild(childName).estimateTreeSize()); } @Override From 65dcb08d7c5103e25c6fa70653e2b843c11b3a2b Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 11:43:16 +0800 Subject: [PATCH 063/109] refactor --- .../schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 1 + .../schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java | 1 + 2 files changed, 2 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 25ea90d1b584..054860e854ad 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem; import org.apache.iotdb.commons.conf.IoTDBConstant; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java index 3ee035a67101..c8b646d0f437 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree; import org.apache.iotdb.commons.conf.IoTDBConstant; From 8570c88ee77122cc0e832b9eac87ef77eac06ed2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 11:54:07 +0800 Subject: [PATCH 064/109] Create DeleteTableDeviceNode.java --- .../node/schema/DeleteTableDeviceNode.java | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java new file mode 100644 index 000000000000..e41de83ff56d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; + +public class DeleteTableDeviceNode extends PlanNode { + private final String tableName; + + public DeleteTableDeviceNode(final PlanNodeId id, final String tableName) { + super(id); + this.tableName = tableName; + } + + public String getTableName() { + return tableName; + } + + @Override + public List getChildren() { + return Collections.emptyList(); + } + + @Override + public void addChild(PlanNode child) { + throw new UnsupportedOperationException(); + } + + @Override + public PlanNode clone() { + return new DeleteTableDeviceNode(id, tableName); + } + + @Override + public int allowedChildCount() { + return 0; + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + public PlanNodeType getType() { + return null; + } + + @Override + protected void serializeAttributes(final ByteBuffer byteBuffer) { + getType().serialize(byteBuffer); + ReadWriteIOUtils.write(tableName, byteBuffer); + } + + @Override + protected void serializeAttributes(final DataOutputStream stream) throws IOException { + getType().serialize(stream); + ReadWriteIOUtils.write(tableName, stream); + } + + public static DeleteTableDeviceNode deserialize(final ByteBuffer buffer) { + final String tableName = ReadWriteIOUtils.readString(buffer); + final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); + return new DeleteTableDeviceNode(planNodeId, tableName); + } +} From e45c8e8de49f9f91c761fe6a30523028548a8b61 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 12:12:14 +0800 Subject: [PATCH 065/109] Partial --- .../plan/planner/plan/node/PlanNodeType.java | 3 ++ .../node/schema/DeleteTableDeviceNode.java | 17 +++++++-- .../schemaregion/SchemaRegionPlanType.java | 13 +++---- .../schemaregion/SchemaRegionPlanVisitor.java | 6 ++++ .../visitor/SchemaRegionPlanDeserializer.java | 7 ++++ .../visitor/SchemaRegionPlanSerializer.java | 36 +++++++++---------- 6 files changed, 55 insertions(+), 27 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index d59543f827a8..6c055d99f46e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -120,6 +120,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; @@ -542,6 +543,8 @@ public static PlanNode deserialize(ByteBuffer buffer, short nodeType) { return TableDeviceQueryScanNode.deserialize(buffer); case 904: return TableDeviceFetchNode.deserialize(buffer); + case 905: + return DeleteTableDeviceNode.deserialize(buffer); case 906: return TableDeviceQueryCountNode.deserialize(buffer); case 907: diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java index e41de83ff56d..c6cb4cea4726 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java @@ -22,6 +22,9 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; +import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; +import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -31,7 +34,7 @@ import java.util.Collections; import java.util.List; -public class DeleteTableDeviceNode extends PlanNode { +public class DeleteTableDeviceNode extends PlanNode implements ISchemaRegionPlan { private final String tableName; public DeleteTableDeviceNode(final PlanNodeId id, final String tableName) { @@ -70,7 +73,7 @@ public List getOutputColumnNames() { @Override public PlanNodeType getType() { - return null; + return PlanNodeType.DELETE_TABLE_DEVICE; } @Override @@ -90,4 +93,14 @@ public static DeleteTableDeviceNode deserialize(final ByteBuffer buffer) { final PlanNodeId planNodeId = PlanNodeId.deserialize(buffer); return new DeleteTableDeviceNode(planNodeId, tableName); } + + @Override + public SchemaRegionPlanType getPlanType() { + return SchemaRegionPlanType.DELETE_TABLE_DEVICE; + } + + @Override + public R accept(final SchemaRegionPlanVisitor visitor, final C context) { + return visitor.visitDeleteTableDevice(this, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java index 0553e3bf23bd..0be34b48dfb9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanType.java @@ -56,6 +56,7 @@ public enum SchemaRegionPlanType { UPDATE_TABLE_DEVICE_ATTRIBUTE((byte) 101), COMMIT_UPDATE_TABLE_DEVICE_ATTRIBUTE((byte) 102), ADD_NODE_LOCATION((byte) 103), + DELETE_TABLE_DEVICE((byte) 104), // query plan doesn't need any ser/deSer, thus use one type to represent all READ_SCHEMA(Byte.MAX_VALUE); @@ -64,14 +65,14 @@ public enum SchemaRegionPlanType { private static final SchemaRegionPlanType[] PLAN_TYPE_TABLE = new SchemaRegionPlanType[MAX_NUM]; static { - for (SchemaRegionPlanType type : SchemaRegionPlanType.values()) { + for (final SchemaRegionPlanType type : SchemaRegionPlanType.values()) { PLAN_TYPE_TABLE[type.planType] = type; } } private final byte planType; - SchemaRegionPlanType(byte planType) { + SchemaRegionPlanType(final byte planType) { this.planType = planType; } @@ -79,13 +80,13 @@ public byte getPlanType() { return planType; } - public void serialize(DataOutputStream dataOutputStream) throws IOException { + public void serialize(final DataOutputStream dataOutputStream) throws IOException { dataOutputStream.writeByte(planType); } - public static SchemaRegionPlanType deserialize(ByteBuffer buffer) { - byte code = buffer.get(); - SchemaRegionPlanType type = PLAN_TYPE_TABLE[code]; + public static SchemaRegionPlanType deserialize(final ByteBuffer buffer) { + final byte code = buffer.get(); + final SchemaRegionPlanType type = PLAN_TYPE_TABLE[code]; if (type == null) { throw new IllegalArgumentException("Unrecognized SchemaRegionPlanType of " + code); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java index 1b108bd99b82..d34d54cd9628 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/SchemaRegionPlanVisitor.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; @@ -148,4 +149,9 @@ public R visitAddNodeLocation( final TableNodeLocationAddNode addNodeLocationPlan, final C context) { return visitSchemaRegionPlan(addNodeLocationPlan, context); } + + public R visitDeleteTableDevice( + final DeleteTableDeviceNode deleteTableDevicePlan, final C context) { + return visitSchemaRegionPlan(deleteTableDevicePlan, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java index 0d9e44a8c4f6..3062792bb93c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanDeserializer.java @@ -26,6 +26,7 @@ import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; @@ -437,5 +438,11 @@ public ISchemaRegionPlan visitAddNodeLocation( final TableNodeLocationAddNode addNodeLocationPlan, final ByteBuffer buffer) { return (TableNodeLocationAddNode) PlanNodeType.deserialize(buffer); } + + @Override + public ISchemaRegionPlan visitDeleteTableDevice( + final DeleteTableDeviceNode addNodeLocationPlan, final ByteBuffer buffer) { + return (DeleteTableDeviceNode) PlanNodeType.deserialize(buffer); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java index 7f5ee7c61e93..128b20f17bef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/logfile/visitor/SchemaRegionPlanSerializer.java @@ -21,7 +21,9 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; @@ -482,43 +484,39 @@ public SchemaRegionPlanSerializationResult visitDeleteLogicalView( public SchemaRegionPlanSerializationResult visitCreateOrUpdateTableDevice( final CreateOrUpdateTableDeviceNode createOrUpdateTableDeviceNode, final DataOutputStream outputStream) { - try { - createOrUpdateTableDeviceNode.serialize(outputStream); - return SchemaRegionPlanSerializationResult.SUCCESS; - } catch (final IOException e) { - return new SchemaRegionPlanSerializationResult(e); - } + return visitPlanNode(createOrUpdateTableDeviceNode, outputStream); } @Override public SchemaRegionPlanSerializationResult visitUpdateTableDeviceAttribute( final TableDeviceAttributeUpdateNode updateTableDeviceAttributePlan, final DataOutputStream outputStream) { - try { - updateTableDeviceAttributePlan.serialize(outputStream); - return SchemaRegionPlanSerializationResult.SUCCESS; - } catch (final IOException e) { - return new SchemaRegionPlanSerializationResult(e); - } + return visitPlanNode(updateTableDeviceAttributePlan, outputStream); } @Override public SchemaRegionPlanSerializationResult visitCommitUpdateTableDeviceAttribute( final TableDeviceAttributeCommitUpdateNode commitUpdateTableDeviceAttributePlan, final DataOutputStream outputStream) { - try { - commitUpdateTableDeviceAttributePlan.serialize(outputStream); - return SchemaRegionPlanSerializationResult.SUCCESS; - } catch (final IOException e) { - return new SchemaRegionPlanSerializationResult(e); - } + return visitPlanNode(commitUpdateTableDeviceAttributePlan, outputStream); } @Override public SchemaRegionPlanSerializationResult visitAddNodeLocation( final TableNodeLocationAddNode addNodeLocationPlan, final DataOutputStream outputStream) { + return visitPlanNode(addNodeLocationPlan, outputStream); + } + + @Override + public SchemaRegionPlanSerializationResult visitDeleteTableDevice( + final DeleteTableDeviceNode deleteTableDevicePlan, final DataOutputStream outputStream) { + return visitPlanNode(deleteTableDevicePlan, outputStream); + } + + private SchemaRegionPlanSerializationResult visitPlanNode( + final PlanNode planNode, final DataOutputStream outputStream) { try { - addNodeLocationPlan.serialize(outputStream); + planNode.serialize(outputStream); return SchemaRegionPlanSerializationResult.SUCCESS; } catch (final IOException e) { return new SchemaRegionPlanSerializationResult(e); From cef77f3a1bafc7ae36ab7bed47c4981b290a2129 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 12:15:20 +0800 Subject: [PATCH 066/109] Next --- .../relational/planner/node/schema/DeleteTableDeviceNode.java | 2 ++ .../schemaregion/write/req/SchemaRegionWritePlanFactory.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java index c6cb4cea4726..185003edbfdc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java @@ -36,6 +36,8 @@ public class DeleteTableDeviceNode extends PlanNode implements ISchemaRegionPlan { private final String tableName; + public static final DeleteTableDeviceNode MOCK_INSTANCE = + new DeleteTableDeviceNode(new PlanNodeId(""), null); public DeleteTableDeviceNode(final PlanNodeId id, final String tableName) { super(id); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java index 5c057be3247e..f4d1fe5515db 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/write/req/SchemaRegionWritePlanFactory.java @@ -23,6 +23,7 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; @@ -106,6 +107,8 @@ public static ISchemaRegionPlan getEmptyPlan(final SchemaRegionPlanType planType return TableDeviceAttributeCommitUpdateNode.MOCK_INSTANCE; case ADD_NODE_LOCATION: return TableNodeLocationAddNode.MOCK_INSTANCE; + case DELETE_TABLE_DEVICE: + return DeleteTableDeviceNode.MOCK_INSTANCE; default: throw new UnsupportedOperationException( String.format( From 016a56939775a01f8eff149d3261b4724412094e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 12:17:24 +0800 Subject: [PATCH 067/109] Continue --- .../iotdb/db/schemaengine/schemaregion/ISchemaRegion.java | 4 +++- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 8 ++++++-- .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 3 ++- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java index e6e77d65ce37..5787547ae56b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/ISchemaRegion.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; @@ -334,7 +335,8 @@ void createOrUpdateTableDevice(final CreateOrUpdateTableDeviceNode createOrUpdat void updateTableDeviceAttribute(final TableDeviceAttributeUpdateNode updateNode) throws MetadataException; - void deleteTableDevice(final String table) throws MetadataException; + void deleteTableDevice(final DeleteTableDeviceNode deleteTableDeviceNode) + throws MetadataException; // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 28bc2101edb7..dc82878805c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -53,6 +53,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; @@ -1518,8 +1519,11 @@ private DeviceAttributeUpdater constructDevicePredicateUpdater( } @Override - public void deleteTableDevice(final String table) { - mtree.deleteTableDevice(table); + public void deleteTableDevice(final DeleteTableDeviceNode deleteTableDeviceNode) + throws MetadataException { + if (mtree.deleteTableDevice(deleteTableDeviceNode.getTableName())) { + writeToMLog(deleteTableDeviceNode); + } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 11c2461a326f..501c183bebf5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.queryengine.common.schematree.ClusterSchemaTree; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; @@ -1445,7 +1446,7 @@ public void updateTableDeviceAttribute(final TableDeviceAttributeUpdateNode upda } @Override - public void deleteTableDevice(String table) { + public void deleteTableDevice(final DeleteTableDeviceNode deleteTableDeviceNode) { throw new UnsupportedOperationException("TableModel does not support PBTree yet."); } From 82fcf1cec4c550bcae0dc52986b428b9017679a6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 12:21:15 +0800 Subject: [PATCH 068/109] Next --- .../schemaregion/SchemaExecutionVisitor.java | 13 +++++++++++++ .../plan/planner/plan/node/PlanVisitor.java | 5 +++++ .../planner/node/schema/DeleteTableDeviceNode.java | 6 ++++++ 3 files changed, 24 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java index 0d6fd015ba92..3e437f72a37a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/statemachine/schemaregion/SchemaExecutionVisitor.java @@ -58,6 +58,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeEnrichedWritePlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.pipe.PipeOperateSchemaQueueNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableNodeLocationAddNode; @@ -630,6 +631,18 @@ public TSStatus visitTableNodeLocationAdd( } } + @Override + public TSStatus visitDeleteTableDevice( + final DeleteTableDeviceNode node, final ISchemaRegion schemaRegion) { + try { + schemaRegion.deleteTableDevice(node); + return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); + } catch (final MetadataException e) { + logger.error(e.getMessage(), e); + return RpcUtils.getStatus(e.getErrorCode(), e.getMessage()); + } + } + @Override public TSStatus visitPipeEnrichedWritePlanNode( final PipeEnrichedWritePlanNode node, final ISchemaRegion schemaRegion) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index f0f877d30db3..04ab4bbafc10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -126,6 +126,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.CreateOrUpdateTableDeviceNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeCommitUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceAttributeUpdateNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableDeviceFetchNode; @@ -539,6 +540,10 @@ public R visitTableNodeLocationAdd(final TableNodeLocationAddNode node, final C return visitPlan(node, context); } + public R visitDeleteTableDevice(final DeleteTableDeviceNode node, final C context) { + return visitPlan(node, context); + } + ///////////////////////////////////////////////////////////////////////////////////////////////// // Data Write Node ///////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java index 185003edbfdc..e269a4ea5a1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.ISchemaRegionPlan; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanType; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegionPlanVisitor; @@ -96,6 +97,11 @@ public static DeleteTableDeviceNode deserialize(final ByteBuffer buffer) { return new DeleteTableDeviceNode(planNodeId, tableName); } + @Override + public R accept(final PlanVisitor visitor, C context) { + return visitor.visitDeleteTableDevice(this, context); + } + @Override public SchemaRegionPlanType getPlanType() { return SchemaRegionPlanType.DELETE_TABLE_DEVICE; From f8052aacc632ea40a523b712a12109f8045a93fd Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 12:23:30 +0800 Subject: [PATCH 069/109] Update SchemaRegionMemoryImpl.java --- .../schemaregion/impl/SchemaRegionMemoryImpl.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index dc82878805c6..8fe129e2ed1d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1856,5 +1856,16 @@ public RecoverOperationResult visitAddNodeLocation( return new RecoverOperationResult(e); } } + + @Override + public RecoverOperationResult visitDeleteTableDevice( + final DeleteTableDeviceNode deleteTableDevicePlan, final SchemaRegionMemoryImpl context) { + try { + deleteTableDevice(deleteTableDevicePlan); + return RecoverOperationResult.SUCCESS; + } catch (final MetadataException e) { + return new RecoverOperationResult(e); + } + } } } From 80be63b9319aed3cca6fd096423a5d36adf40dc7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 12:41:48 +0800 Subject: [PATCH 070/109] Enrich --- ...oDnInternalServiceAsyncRequestManager.java | 9 +++- .../impl/schema/table/DropTableProcedure.java | 49 +++++++++++++++++-- .../impl/DataNodeInternalRPCServiceImpl.java | 16 ++++-- .../src/main/thrift/datanode.thrift | 16 +++--- 4 files changed, 74 insertions(+), 16 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java index ef44d6eaa840..a4408f98d554 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java @@ -62,7 +62,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TCreateTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TDeactivateTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDeleteSchemaReq; -import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDropTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataOrDevicesForDropTableReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteTimeSeriesReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteViewSchemaReq; import org.apache.iotdb.mpp.rpc.thrift.TDropFunctionInstanceReq; @@ -380,7 +380,12 @@ protected void initActionMapBuilder() { CnToDnAsyncRequestType.DELETE_DATA_FOR_DROP_TABLE, (req, client, handler) -> client.deleteDataForDropTable( - (TDeleteDataForDropTableReq) req, (DataNodeTSStatusRPCHandler) handler)); + (TDeleteDataOrDevicesForDropTableReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.DELETE_DEVICES_FOR_DROP_TABLE, + (req, client, handler) -> + client.deleteDevicesForDropTable( + (TDeleteDataOrDevicesForDropTableReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( CnToDnAsyncRequestType.CLEAN_DATA_NODE_CACHE, (req, client, handler) -> diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 2b847bd70287..6359146dc95d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -37,7 +37,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeRegionTaskExecutor; import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; -import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDropTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataOrDevicesForDropTableReq; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; @@ -49,6 +49,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.BiFunction; @@ -59,6 +60,9 @@ public class DropTableProcedure extends AbstractAlterOrDropTableProcedure( - "delete data", + "delete data for drop table", env, relatedDataRegionGroup, true, CnToDnAsyncRequestType.DELETE_DATA_FOR_DROP_TABLE, ((dataNodeLocation, consensusGroupIdList) -> - new TDeleteDataForDropTableReq( - new ArrayList<>(consensusGroupIdList), database, tableName))); + new TDeleteDataOrDevicesForDropTableReq( + new ArrayList<>(consensusGroupIdList), tableName))); + } + + private void deleteSchema(final ConfigNodeProcedureEnv env) { + if (Objects.isNull(patternTree)) { + patternTree = new PathPatternTree(); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); + final PartialPath path; + try { + path = new PartialPath(new String[] {ROOT, database, table.getTableName()}); + patternTree.appendPathPattern(path); + patternTree.appendPathPattern(path.concatAsMeasurementPath(MULTI_LEVEL_PATH_WILDCARD)); + patternTree.serialize(dataOutputStream); + } catch (final IOException e) { + LOGGER.warn( + "failed to serialize request for table {}.{}", database, table.getTableName(), e); + } + } + + final Map relatedSchemaRegionGroup = + env.getConfigManager().getRelatedSchemaRegionGroup(patternTree); + + if (relatedSchemaRegionGroup.isEmpty()) { + return; + } + + new DropTableRegionTaskExecutor<>( + "delete devices for drop table", + env, + relatedSchemaRegionGroup, + true, + CnToDnAsyncRequestType.DELETE_DEVICES_FOR_DROP_TABLE, + ((dataNodeLocation, consensusGroupIdList) -> + new TDeleteDataOrDevicesForDropTableReq( + new ArrayList<>(consensusGroupIdList), tableName))); } private void dropTable(final ConfigNodeProcedureEnv env) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 2f90c5675847..31d429e0454a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -196,7 +196,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp; import org.apache.iotdb.mpp.rpc.thrift.TDeactivateTemplateReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDeleteSchemaReq; -import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataForDropTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataOrDevicesForDropTableReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteTimeSeriesReq; import org.apache.iotdb.mpp.rpc.thrift.TDeleteViewSchemaReq; import org.apache.iotdb.mpp.rpc.thrift.TDropFunctionInstanceReq; @@ -1545,9 +1545,19 @@ public TSStatus updateTable(final TUpdateTableReq req) { } @Override - public TSStatus deleteDataForDropTable(final TDeleteDataForDropTableReq req) { + public TSStatus deleteDataForDropTable(final TDeleteDataOrDevicesForDropTableReq req) { return executeInternalSchemaTask( - req.getDataRegionIdList(), + req.getRegionIdList(), + consensusGroupId -> { + // TODO + return StatusUtils.OK; + }); + } + + @Override + public TSStatus deleteDevicesForDropTable(final TDeleteDataOrDevicesForDropTableReq req) { + return executeInternalSchemaTask( + req.getRegionIdList(), consensusGroupId -> { // TODO return StatusUtils.OK; diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index faed6e33f594..49a1637245ec 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -330,10 +330,9 @@ struct TUpdateTableReq { 2: required binary tableInfo } -struct TDeleteDataForDropTableReq { - 1: required list dataRegionIdList - 2: required string database - 3: required string tableName +struct TDeleteDataOrDevicesForDropTableReq { + 1: required list regionIdList + 2: required string tableName } struct TTsFilePieceReq { @@ -1064,9 +1063,14 @@ service IDataNodeRPCService { common.TSStatus updateTable(TUpdateTableReq req) /** - * Delete data for delete table + * Delete data for drop table + */ + common.TSStatus deleteDataForDropTable(TDeleteDataOrDevicesForDropTableReq req) + + /** + * Delete devices for drop table */ - common.TSStatus deleteDataForDropTable(TDeleteDataForDropTableReq req) + common.TSStatus deleteDevicesForDropTable(TDeleteDataOrDevicesForDropTableReq req) common.TTestConnectionResp submitTestConnectionTask(common.TNodeLocations nodeLocations) From def18359912dcf862c9d4e5016358bcb0c743f97 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:09:50 +0800 Subject: [PATCH 071/109] Update DataNodeInternalRPCServiceImpl.java --- .../thrift/impl/DataNodeInternalRPCServiceImpl.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index 31d429e0454a..e7982975bc43 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -141,6 +141,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableDeviceSchemaFetcher; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TableDeviceSchemaCache; import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.TreeDeviceSchemaCacheManager; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.DeleteTableDeviceNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.schema.TableSchemaQueryWriteVisitor; import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler; import org.apache.iotdb.db.queryengine.plan.statement.component.WhereCondition; @@ -1558,10 +1559,12 @@ public TSStatus deleteDataForDropTable(final TDeleteDataOrDevicesForDropTableReq public TSStatus deleteDevicesForDropTable(final TDeleteDataOrDevicesForDropTableReq req) { return executeInternalSchemaTask( req.getRegionIdList(), - consensusGroupId -> { - // TODO - return StatusUtils.OK; - }); + consensusGroupId -> + new RegionWriteExecutor() + .execute( + new SchemaRegionId(consensusGroupId.getId()), + new DeleteTableDeviceNode(new PlanNodeId(""), req.getTableName())) + .getStatus()); } public TTestConnectionResp submitTestConnectionTask(TNodeLocations nodeLocations) { From fba973c5c73a354acfdd39a3b42a21f72238110a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:11:14 +0800 Subject: [PATCH 072/109] Update DropTableProcedure.java --- .../procedure/impl/schema/table/DropTableProcedure.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 6359146dc95d..9e89b807bfce 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -97,11 +97,12 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DropTabl break; case DELETE_DEVICES: LOGGER.info("Deleting devices for table {}.{} when dropping table", database, tableName); - return Flow.NO_MORE_STATE; + deleteSchema(env); + break; case DROP_TABLE: LOGGER.info("Dropping table {}.{} on configNode", database, tableName); dropTable(env); - break; + return Flow.NO_MORE_STATE; default: setFailure(new ProcedureException("Unrecognized DropTableState " + state)); return Flow.NO_MORE_STATE; From b18643398d1cb187e753b05db0744a533c2c8620 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:40:35 +0800 Subject: [PATCH 073/109] invalidate cache --- .../client/async/CnToDnAsyncRequestType.java | 1 + .../impl/DataNodeInternalRPCServiceImpl.java | 11 +++++++++++ .../fetcher/cache/TableDeviceSchemaCache.java | 15 ++++++++++++++- .../schemaengine/table/DataNodeTableCache.java | 18 ++++++++++++++++++ .../db/schemaengine/table/ITableCache.java | 2 ++ .../src/main/thrift/datanode.thrift | 5 +++++ 6 files changed, 51 insertions(+), 1 deletion(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java index db9066241b16..8404db8101c0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnAsyncRequestType.java @@ -110,6 +110,7 @@ public enum CnToDnAsyncRequestType { // Table UPDATE_TABLE, + INVALIDATE_TABLE_CACHE, DELETE_DATA_FOR_DROP_TABLE, DELETE_DEVICES_FOR_DROP_TABLE, } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index e7982975bc43..ad1dd0c0264c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -1545,6 +1545,17 @@ public TSStatus updateTable(final TUpdateTableReq req) { return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS); } + @Override + public TSStatus invalidateTableCache(final String database, final String tableName) { + DataNodeSchemaLockManager.getInstance().takeWriteLock(SchemaLockType.VALIDATE_VS_DELETION); + try { + TableDeviceSchemaCache.getInstance().invalidate(database, tableName); + return StatusUtils.OK; + } finally { + DataNodeSchemaLockManager.getInstance().releaseWriteLock(SchemaLockType.VALIDATE_VS_DELETION); + } + } + @Override public TSStatus deleteDataForDropTable(final TDeleteDataOrDevicesForDropTableReq req) { return executeInternalSchemaTask( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java index 7acd9dcb3b37..52ca5ccbea09 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.dualkeycache.impl.DualKeyCachePolicy; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.schemaengine.schemaregion.SchemaRegion; +import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.StringArrayDeviceID; @@ -134,6 +135,11 @@ public void putAttributes( final String database, final IDeviceID deviceId, final Map attributeMap) { readWriteLock.readLock().lock(); try { + // Avoid stale table + if (Objects.isNull( + DataNodeTableCache.getInstance().getTable(database, deviceId.getTableName()))) { + return; + } dualKeyCache.update( new TableId(database, deviceId.getTableName()), deviceId, @@ -215,6 +221,11 @@ public void updateLastCache( final @Nullable TimeValuePair[] timeValuePairs) { readWriteLock.readLock().lock(); try { + // Avoid stale table + if (Objects.isNull( + DataNodeTableCache.getInstance().getTable(database, deviceId.getTableName()))) { + return; + } dualKeyCache.update( new TableId(database, deviceId.getTableName()), deviceId, @@ -252,7 +263,7 @@ public void updateLastCacheIfExists( } /** - * Get the last entry of a measurement, the measurement shall never be "time". + * Get the last {@link TimeValuePair} of a measurement, the measurement shall never be "time". * * @param database the device's database, without "root", {@code null} for tree model * @param deviceId {@link IDeviceID} @@ -514,6 +525,8 @@ public void invalidate(final @Nonnull String database) { public void invalidate(final String database, final String tableName) { readWriteLock.writeLock().lock(); try { + // Table cache's invalidate must be guarded by this lock + DataNodeTableCache.getInstance().invalid(database, tableName); dualKeyCache.invalidate(new TableId(database, tableName)); } finally { readWriteLock.writeLock().unlock(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java index e99df7ccd365..9b8ed5ddbdf1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/DataNodeTableCache.java @@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; +import javax.annotation.concurrent.GuardedBy; import java.util.Collections; import java.util.List; @@ -190,6 +191,23 @@ public void invalid(String database) { } } + @GuardedBy("TableDeviceSchemaCache#writeLock") + @Override + public void invalid(String database, final String tableName) { + database = PathUtils.unQualifyDatabaseName(database); + readWriteLock.writeLock().lock(); + try { + if (databaseTableMap.containsKey(database)) { + databaseTableMap.get(database).remove(tableName); + } + if (preUpdateTableMap.containsKey(database)) { + preUpdateTableMap.get(database).remove(tableName); + } + } finally { + readWriteLock.writeLock().unlock(); + } + } + public TsTable getTableInWrite(final String database, final String tableName) { final TsTable result = getTableInCache(database, tableName); return Objects.nonNull(result) ? result : getTable(database, tableName); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java index fbc50d6f4deb..c9671f7cde5c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/table/ITableCache.java @@ -35,4 +35,6 @@ public interface ITableCache { * @param database shouldn't start with `root.` */ void invalid(final String database); + + void invalid(final String database, final String tableName); } diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 49a1637245ec..129002dcbab2 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -1062,6 +1062,11 @@ service IDataNodeRPCService { */ common.TSStatus updateTable(TUpdateTableReq req) + /** + * Delete data for drop table, this database is without "root" + */ + common.TSStatus invalidateTableCache(string database, string tableName) + /** * Delete data for drop table */ From a853f924b55e84078e12fd4dd429de04597a8862 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:43:32 +0800 Subject: [PATCH 074/109] Update DropTableProcedure.java --- .../procedure/impl/schema/table/DropTableProcedure.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 9e89b807bfce..625a415d3897 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -135,7 +135,7 @@ private void deleteData(final ConfigNodeProcedureEnv env) { final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); final PartialPath path; try { - path = new PartialPath(new String[] {ROOT, database, table.getTableName()}); + path = new PartialPath(new String[] {ROOT, database, tableName}); patternTree.appendPathPattern(path); patternTree.appendPathPattern(path.concatAsMeasurementPath(MULTI_LEVEL_PATH_WILDCARD)); patternTree.serialize(dataOutputStream); @@ -168,7 +168,7 @@ private void deleteSchema(final ConfigNodeProcedureEnv env) { final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); final PartialPath path; try { - path = new PartialPath(new String[] {ROOT, database, table.getTableName()}); + path = new PartialPath(new String[] {ROOT, database, tableName}); patternTree.appendPathPattern(path); patternTree.appendPathPattern(path.concatAsMeasurementPath(MULTI_LEVEL_PATH_WILDCARD)); patternTree.serialize(dataOutputStream); From 7f1c93f451345ae2f7772f406f06246d01054cf1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:45:29 +0800 Subject: [PATCH 075/109] Update DropTableProcedure.java --- .../impl/schema/table/DropTableProcedure.java | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 625a415d3897..80a064f6ff6e 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -151,14 +151,17 @@ private void deleteData(final ConfigNodeProcedureEnv env) { } new DropTableRegionTaskExecutor<>( - "delete data for drop table", - env, - relatedDataRegionGroup, - true, - CnToDnAsyncRequestType.DELETE_DATA_FOR_DROP_TABLE, - ((dataNodeLocation, consensusGroupIdList) -> - new TDeleteDataOrDevicesForDropTableReq( - new ArrayList<>(consensusGroupIdList), tableName))); + "delete data for drop table", + env, + relatedDataRegionGroup, + true, + CnToDnAsyncRequestType.DELETE_DATA_FOR_DROP_TABLE, + ((dataNodeLocation, consensusGroupIdList) -> + new TDeleteDataOrDevicesForDropTableReq( + new ArrayList<>(consensusGroupIdList), tableName))) + .execute(); + + setNextState(DropTableState.DELETE_DEVICES); } private void deleteSchema(final ConfigNodeProcedureEnv env) { @@ -186,14 +189,17 @@ private void deleteSchema(final ConfigNodeProcedureEnv env) { } new DropTableRegionTaskExecutor<>( - "delete devices for drop table", - env, - relatedSchemaRegionGroup, - true, - CnToDnAsyncRequestType.DELETE_DEVICES_FOR_DROP_TABLE, - ((dataNodeLocation, consensusGroupIdList) -> - new TDeleteDataOrDevicesForDropTableReq( - new ArrayList<>(consensusGroupIdList), tableName))); + "delete devices for drop table", + env, + relatedSchemaRegionGroup, + true, + CnToDnAsyncRequestType.DELETE_DEVICES_FOR_DROP_TABLE, + ((dataNodeLocation, consensusGroupIdList) -> + new TDeleteDataOrDevicesForDropTableReq( + new ArrayList<>(consensusGroupIdList), tableName))) + .execute(); + + setNextState(DropTableState.DROP_TABLE); } private void dropTable(final ConfigNodeProcedureEnv env) { From 482fc8e361575254faa599cf04a3511feb181eb7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:47:49 +0800 Subject: [PATCH 076/109] Update DropTableProcedure.java --- .../impl/schema/table/DropTableProcedure.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 80a064f6ff6e..412d03f7fc53 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -28,6 +28,8 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType; +import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager; +import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext; import org.apache.iotdb.confignode.consensus.request.write.table.DropTablePlan; import org.apache.iotdb.confignode.consensus.request.write.table.PreDeleteTablePlan; import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv; @@ -38,6 +40,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataOrDevicesForDropTableReq; +import org.apache.iotdb.mpp.rpc.thrift.TInvalidateMatchedSchemaCacheReq; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; @@ -90,6 +93,7 @@ protected Flow executeFromState(final ConfigNodeProcedureEnv env, final DropTabl case INVALIDATE_CACHE: LOGGER.info( "Invalidating cache for table {}.{} when invalidating cache", database, tableName); + invalidateCache(env); break; case DELETE_DATA: LOGGER.info("Deleting data for table {}.{}", database, tableName); @@ -129,6 +133,29 @@ private void checkAndPreDeleteTable(final ConfigNodeProcedureEnv env) { } } + private void invalidateCache(final ConfigNodeProcedureEnv env) { + final Map dataNodeLocationMap = + env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); + final DataNodeAsyncRequestContext clientHandler = + new DataNodeAsyncRequestContext<>( + CnToDnAsyncRequestType.INVALIDATE_TABLE_CACHE, + new TInvalidateMatchedSchemaCacheReq(patternTreeBytes), + dataNodeLocationMap); + CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler); + final Map statusMap = clientHandler.getResponseMap(); + for (final TSStatus status : statusMap.values()) { + // All dataNodes must clear the related schemaEngine cache + if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { + LOGGER.error("Failed to invalidate schemaEngine cache of table {}.{}", database, tableName); + setFailure( + new ProcedureException(new MetadataException("Invalidate schemaEngine cache failed"))); + return; + } + } + + setNextState(DropTableState.DELETE_DATA); + } + private void deleteData(final ConfigNodeProcedureEnv env) { patternTree = new PathPatternTree(); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); From 1f9c04337161ab61a05a15178117f0da5825ab02 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:50:36 +0800 Subject: [PATCH 077/109] Update datanode.thrift --- .../thrift-datanode/src/main/thrift/datanode.thrift | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift index 129002dcbab2..b79cce400b02 100644 --- a/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift +++ b/iotdb-protocol/thrift-datanode/src/main/thrift/datanode.thrift @@ -330,6 +330,11 @@ struct TUpdateTableReq { 2: required binary tableInfo } +struct TInvalidateTableCacheReq { + 1: required string database, + 2: required string tableName +} + struct TDeleteDataOrDevicesForDropTableReq { 1: required list regionIdList 2: required string tableName @@ -1065,7 +1070,7 @@ service IDataNodeRPCService { /** * Delete data for drop table, this database is without "root" */ - common.TSStatus invalidateTableCache(string database, string tableName) + common.TSStatus invalidateTableCache(TInvalidateTableCacheReq req) /** * Delete data for drop table From 39d35c4df246c9a3e24dc1cff26734278dfda2ab Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:55:33 +0800 Subject: [PATCH 078/109] Invalidate cache --- .../async/CnToDnInternalServiceAsyncRequestManager.java | 6 ++++++ .../procedure/impl/schema/table/DropTableProcedure.java | 6 +++--- .../thrift/impl/DataNodeInternalRPCServiceImpl.java | 5 +++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java index a4408f98d554..70c168574b19 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/CnToDnInternalServiceAsyncRequestManager.java @@ -72,6 +72,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TInactiveTriggerInstanceReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateMatchedSchemaCacheReq; +import org.apache.iotdb.mpp.rpc.thrift.TInvalidateTableCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TPipeHeartbeatReq; import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaReq; import org.apache.iotdb.mpp.rpc.thrift.TPushMultiPipeMetaReq; @@ -376,6 +377,11 @@ protected void initActionMapBuilder() { CnToDnAsyncRequestType.UPDATE_TABLE, (req, client, handler) -> client.updateTable((TUpdateTableReq) req, (DataNodeTSStatusRPCHandler) handler)); + actionMapBuilder.put( + CnToDnAsyncRequestType.INVALIDATE_TABLE_CACHE, + (req, client, handler) -> + client.invalidateTableCache( + (TInvalidateTableCacheReq) req, (DataNodeTSStatusRPCHandler) handler)); actionMapBuilder.put( CnToDnAsyncRequestType.DELETE_DATA_FOR_DROP_TABLE, (req, client, handler) -> diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 412d03f7fc53..39c0b1af5333 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -40,7 +40,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataOrDevicesForDropTableReq; -import org.apache.iotdb.mpp.rpc.thrift.TInvalidateMatchedSchemaCacheReq; +import org.apache.iotdb.mpp.rpc.thrift.TInvalidateTableCacheReq; import org.apache.iotdb.rpc.TSStatusCode; import org.slf4j.Logger; @@ -136,10 +136,10 @@ private void checkAndPreDeleteTable(final ConfigNodeProcedureEnv env) { private void invalidateCache(final ConfigNodeProcedureEnv env) { final Map dataNodeLocationMap = env.getConfigManager().getNodeManager().getRegisteredDataNodeLocations(); - final DataNodeAsyncRequestContext clientHandler = + final DataNodeAsyncRequestContext clientHandler = new DataNodeAsyncRequestContext<>( CnToDnAsyncRequestType.INVALIDATE_TABLE_CACHE, - new TInvalidateMatchedSchemaCacheReq(patternTreeBytes), + new TInvalidateTableCacheReq(database, tableName), dataNodeLocationMap); CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler); final Map statusMap = clientHandler.getResponseMap(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java index ad1dd0c0264c..732274ca8110 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java @@ -216,6 +216,7 @@ import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateMatchedSchemaCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidatePermissionCacheReq; +import org.apache.iotdb.mpp.rpc.thrift.TInvalidateTableCacheReq; import org.apache.iotdb.mpp.rpc.thrift.TLoadCommandReq; import org.apache.iotdb.mpp.rpc.thrift.TLoadResp; import org.apache.iotdb.mpp.rpc.thrift.TMaintainPeerReq; @@ -1546,10 +1547,10 @@ public TSStatus updateTable(final TUpdateTableReq req) { } @Override - public TSStatus invalidateTableCache(final String database, final String tableName) { + public TSStatus invalidateTableCache(final TInvalidateTableCacheReq req) { DataNodeSchemaLockManager.getInstance().takeWriteLock(SchemaLockType.VALIDATE_VS_DELETION); try { - TableDeviceSchemaCache.getInstance().invalidate(database, tableName); + TableDeviceSchemaCache.getInstance().invalidate(req.getDatabase(), req.getTableName()); return StatusUtils.OK; } finally { DataNodeSchemaLockManager.getInstance().releaseWriteLock(SchemaLockType.VALIDATE_VS_DELETION); From 5723362b31aeb2a142ce4634fc43259018ba05f2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:57:46 +0800 Subject: [PATCH 079/109] Update RelationalSql.g4 --- .../org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 4c04f5e2ef6a..1fd34b1948f2 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -166,7 +166,7 @@ dropTableStatement ; showTableStatement - : SHOW TABLES ((FROM | IN) database=identifier)? + : SHOW TABLES (DETAILS)? ((FROM | IN) database=identifier)? // ((LIKE pattern=string (ESCAPE escape=string)) | (WHERE expression))? ; From 7083e7af08c1958af5afebcc7b5a178989bdcfe7 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:15:15 +0800 Subject: [PATCH 080/109] init details --- .../db/queryengine/common/header/ColumnHeaderConstant.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java index ee91da399d51..32bf02aca52f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java @@ -554,6 +554,13 @@ private ColumnHeaderConstant() { ImmutableList.of( new ColumnHeader(TABLE_NAME, TSDataType.TEXT), new ColumnHeader(COLUMN_TTL, TSDataType.TEXT)); + + public static final List showTablesDetailsColumnHeaders = + ImmutableList.of( + new ColumnHeader(TABLE_NAME, TSDataType.TEXT), + new ColumnHeader(COLUMN_TTL, TSDataType.TEXT), + new ColumnHeader(STATUS, TSDataType.TEXT)); + public static final List LIST_USER_PRIVILEGES_Column_HEADERS = ImmutableList.of( new ColumnHeader(ROLE, TSDataType.TEXT), From 456993377c35e4d0640fd5bd8cdee13dc7394ca1 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:16:29 +0800 Subject: [PATCH 081/109] Update ShowTables.java --- .../plan/relational/sql/ast/ShowTables.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowTables.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowTables.java index 776937b9a510..3ed0ff6173e4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowTables.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowTables.java @@ -39,17 +39,17 @@ public ShowTables() { this.dbName = null; } - public ShowTables(NodeLocation location) { + public ShowTables(final NodeLocation location) { super(requireNonNull(location, "location is null")); this.dbName = null; } - public ShowTables(Identifier dbName) { + public ShowTables(final Identifier dbName) { super(null); this.dbName = requireNonNull(dbName, "dbName is null"); } - public ShowTables(NodeLocation location, Identifier dbName) { + public ShowTables(final NodeLocation location, final Identifier dbName) { super(requireNonNull(location, "location is null")); this.dbName = requireNonNull(dbName, "dbName is null"); } @@ -59,7 +59,7 @@ public Optional getDbName() { } @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitShowTables(this, context); } @@ -74,14 +74,14 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if ((obj == null) || (getClass() != obj.getClass())) { return false; } - ShowTables o = (ShowTables) obj; + final ShowTables o = (ShowTables) obj; return Objects.equals(dbName, o.dbName); } From 11a904b4f46b392a22c72006ba5d8afe74c3ee63 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:20:40 +0800 Subject: [PATCH 082/109] Refactor --- .../plan/relational/sql/ast/ShowTables.java | 20 +++++++++---------- .../relational/sql/parser/AstBuilder.java | 7 +++++-- .../relational/sql/util/SqlFormatter.java | 6 +++++- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowTables.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowTables.java index 3ed0ff6173e4..edb711ca9a22 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowTables.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowTables.java @@ -34,30 +34,28 @@ public class ShowTables extends Statement { @Nullable private final Identifier dbName; - public ShowTables() { - super(null); - this.dbName = null; - } + private final boolean isDetails; - public ShowTables(final NodeLocation location) { + public ShowTables(final NodeLocation location, final boolean isDetails) { super(requireNonNull(location, "location is null")); this.dbName = null; + this.isDetails = isDetails; } - public ShowTables(final Identifier dbName) { - super(null); - this.dbName = requireNonNull(dbName, "dbName is null"); - } - - public ShowTables(final NodeLocation location, final Identifier dbName) { + public ShowTables(final NodeLocation location, final Identifier dbName, final boolean isDetails) { super(requireNonNull(location, "location is null")); this.dbName = requireNonNull(dbName, "dbName is null"); + this.isDetails = isDetails; } public Optional getDbName() { return Optional.ofNullable(dbName); } + public boolean isDetails() { + return isDetails; + } + @Override public R accept(final AstVisitor visitor, final C context) { return visitor.visitShowTables(this, context); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index b44efae4966e..b937c1d4619f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -315,8 +315,11 @@ public Node visitDropTableStatement(final RelationalSqlParser.DropTableStatement @Override public Node visitShowTableStatement(final RelationalSqlParser.ShowTableStatementContext ctx) { return Objects.nonNull(ctx.database) - ? new ShowTables(getLocation(ctx), lowerIdentifier((Identifier) visit(ctx.database))) - : new ShowTables(getLocation(ctx)); + ? new ShowTables( + getLocation(ctx), + lowerIdentifier((Identifier) visit(ctx.database)), + Objects.nonNull(ctx.DETAILS())) + : new ShowTables(getLocation(ctx), Objects.nonNull(ctx.DETAILS())); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java index a7756b81163b..97a898d26a25 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java @@ -510,9 +510,13 @@ protected Void visitShowDB(ShowDB node, Integer indent) { } @Override - protected Void visitShowTables(ShowTables node, Integer indent) { + protected Void visitShowTables(final ShowTables node, final Integer indent) { builder.append("SHOW TABLES"); + if (node.isDetails()) { + builder.append(" DETAILS"); + } + node.getDbName().ifPresent(db -> builder.append(" FROM ").append(formatName(db))); return null; From 73cbb029ccd1a18d04e926dd9b872e2712e4500d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:26:31 +0800 Subject: [PATCH 083/109] partial --- .../config/TableConfigTaskVisitor.java | 3 +- .../executor/ClusterConfigTaskExecutor.java | 10 +++-- .../config/executor/IConfigTaskExecutor.java | 2 +- .../relational/ShowTablesDetailsTask.java | 41 +++++++++++++++++++ .../metadata/relational/ShowTablesTask.java | 2 +- 5 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 02d186160397..1f42d8ae9a10 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -49,6 +49,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowConfigNodesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowDataNodesTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowTablesDetailsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowTablesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.UseDBTask; import org.apache.iotdb.db.queryengine.plan.execution.config.sys.FlushTask; @@ -475,7 +476,7 @@ protected IConfigTask visitShowTables(final ShowTables node, final MPPQueryConte if (database == null) { throw new SemanticException(DATABASE_NOT_SPECIFIED); } - return new ShowTablesTask(database); + return node.isDetails() ? new ShowTablesDetailsTask(database) : new ShowTablesTask(database); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 1dd2eb39e63d..a81bab5c5844 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -1300,13 +1300,14 @@ public SettableFuture killQuery(KillQueryStatement killQuerySt } @Override - public SettableFuture showCluster(ShowClusterStatement showClusterStatement) { - SettableFuture future = SettableFuture.create(); + public SettableFuture showCluster( + final ShowClusterStatement showClusterStatement) { + final SettableFuture future = SettableFuture.create(); TShowClusterResp showClusterResp = new TShowClusterResp(); try (ConfigNodeClient client = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { showClusterResp = client.showCluster(); - } catch (ClientManagerException | TException e) { + } catch (final ClientManagerException | TException e) { if (showClusterResp.getConfigNodeList() == null) { future.setException(new TException(MSG_RECONNECTION_FAIL)); } else { @@ -3114,7 +3115,8 @@ public SettableFuture describeTable( } @Override - public SettableFuture showTables(final String database) { + public SettableFuture showTables( + final String database, final boolean isDetails) { final SettableFuture future = SettableFuture.create(); try (final ConfigNodeClient configNodeClient = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java index 41110419332c..4f0b48ba6759 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/IConfigTaskExecutor.java @@ -288,7 +288,7 @@ SettableFuture createTable( SettableFuture describeTable(final String database, final String tableName); - SettableFuture showTables(final String database); + SettableFuture showTables(final String database, final boolean isDetails); TFetchTableResp fetchTables(final Map> fetchTableMap); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java new file mode 100644 index 000000000000..d1bbd2a5486a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; + +import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; +import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; + +import com.google.common.util.concurrent.ListenableFuture; + +public class ShowTablesDetailsTask implements IConfigTask { + + private final String database; + + public ShowTablesDetailsTask(final String database) { + this.database = database; + } + + @Override + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) + throws InterruptedException { + return configTaskExecutor.showTables(database, true); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesTask.java index a37859338b59..c549dd230545 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesTask.java @@ -50,7 +50,7 @@ public ShowTablesTask(final String database) { @Override public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return configTaskExecutor.showTables(database); + return configTaskExecutor.showTables(database, false); } public static void buildTsBlock( From 6e2fe361031423ca8fb32391ae511b8b5305632f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:43:57 +0800 Subject: [PATCH 084/109] Refactor --- .../request/read/table/ShowTablePlan.java | 9 +++- .../confignode/manager/ConfigManager.java | 4 +- .../iotdb/confignode/manager/IManager.java | 2 +- .../manager/schema/ClusterSchemaManager.java | 5 ++- .../persistence/schema/ClusterSchemaInfo.java | 41 +++++++++++++------ .../persistence/schema/ConfigMTree.java | 14 ++++++- .../schema/mnode/impl/ConfigTableNode.java | 1 + .../schema/mnode/info/ConfigTableInfo.java | 2 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 4 +- .../db/protocol/client/ConfigNodeClient.java | 5 ++- .../executor/ClusterConfigTaskExecutor.java | 8 +++- .../schema/table}/TableNodeStatus.java | 8 ++-- .../src/main/thrift/confignode.thrift | 3 +- 13 files changed, 75 insertions(+), 31 deletions(-) rename iotdb-core/{confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl => node-commons/src/main/java/org/apache/iotdb/commons/schema/table}/TableNodeStatus.java (84%) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTablePlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTablePlan.java index 8847151d2115..971a8757372b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTablePlan.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/table/ShowTablePlan.java @@ -26,12 +26,19 @@ public class ShowTablePlan extends ConfigPhysicalReadPlan { private final String database; - public ShowTablePlan(final String database) { + private final boolean isDetails; + + public ShowTablePlan(final String database, final boolean isDetails) { super(ConfigPhysicalPlanType.ShowTable); this.database = database; + this.isDetails = isDetails; } public String getDatabase() { return database; } + + public boolean isDetails() { + return isDetails; + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 57782a8c6255..93e05ed09926 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -2590,10 +2590,10 @@ public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) { } @Override - public TShowTableResp showTables(final String database) { + public TShowTableResp showTables(final String database, final boolean isDetails) { final TSStatus status = confirmLeader(); return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() - ? clusterSchemaManager.showTables(database) + ? clusterSchemaManager.showTables(database, isDetails) : new TShowTableResp(status); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index e46cea8b99b9..1e245baaa1f1 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -834,7 +834,7 @@ TDataPartitionTableResp getOrCreateDataPartition( TSStatus alterOrDropTable(final TAlterOrDropTableReq req); - TShowTableResp showTables(final String database); + TShowTableResp showTables(final String database, final boolean isDetails); TFetchTableResp fetchTables(final Map> fetchTableMap); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java index 037ca7280eb4..c306d8fe1ef2 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java @@ -1066,9 +1066,10 @@ public synchronized TSStatus extendSchemaTemplate( // region table management - public TShowTableResp showTables(final String database) { + public TShowTableResp showTables(final String database, final boolean isDetails) { try { - return ((ShowTableResp) configManager.getConsensusManager().read(new ShowTablePlan(database))) + return ((ShowTableResp) + configManager.getConsensusManager().read(new ShowTablePlan(database, isDetails))) .convertToTShowTableResp(); } catch (final ConsensusException e) { LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index bca28f93ef90..22991eedb2b5 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@ -1112,18 +1112,35 @@ public ShowTableResp showTables(final ShowTablePlan plan) { try { return new ShowTableResp( StatusUtils.OK, - mTree - .getAllUsingTablesUnderSpecificDatabase( - getQualifiedDatabasePartialPath(plan.getDatabase())) - .stream() - .map( - tsTable -> - new TTableInfo( - tsTable.getTableName(), - tsTable - .getPropValue(TTL_PROPERTY.toLowerCase(Locale.ENGLISH)) - .orElse(TTL_INFINITE))) - .collect(Collectors.toList())); + plan.isDetails() + ? mTree + .getAllTablesUnderSpecificDatabase( + getQualifiedDatabasePartialPath(plan.getDatabase())) + .stream() + .map( + pair -> { + final TTableInfo info = + new TTableInfo( + pair.getLeft().getTableName(), + pair.getLeft() + .getPropValue(TTL_PROPERTY.toLowerCase(Locale.ENGLISH)) + .orElse(TTL_INFINITE)); + info.setState(pair.getRight().ordinal()); + return info; + }) + .collect(Collectors.toList()) + : mTree + .getAllUsingTablesUnderSpecificDatabase( + getQualifiedDatabasePartialPath(plan.getDatabase())) + .stream() + .map( + tsTable -> + new TTableInfo( + tsTable.getTableName(), + tsTable + .getPropValue(TTL_PROPERTY.toLowerCase(Locale.ENGLISH)) + .orElse(TTL_INFINITE))) + .collect(Collectors.toList())); } catch (final MetadataException e) { return new ShowTableResp( RpcUtils.getStatus(e.getErrorCode(), e.getMessage()), Collections.emptyList()); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index e504d7e8bfc5..b606bf399268 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -27,13 +27,13 @@ import org.apache.iotdb.commons.schema.node.role.IDatabaseMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.table.TableNodeStatus; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchema; import org.apache.iotdb.commons.utils.ThriftConfigNodeSerDeUtils; import org.apache.iotdb.confignode.persistence.schema.mnode.IConfigMNode; import org.apache.iotdb.confignode.persistence.schema.mnode.factory.ConfigMNodeFactory; import org.apache.iotdb.confignode.persistence.schema.mnode.impl.ConfigTableNode; -import org.apache.iotdb.confignode.persistence.schema.mnode.impl.TableNodeStatus; import org.apache.iotdb.db.exception.metadata.DatabaseAlreadySetException; import org.apache.iotdb.db.exception.metadata.DatabaseConflictException; import org.apache.iotdb.db.exception.metadata.DatabaseNotSetException; @@ -705,6 +705,18 @@ public List getAllUsingTablesUnderSpecificDatabase(final PartialPath da .collect(Collectors.toList()); } + public List> getAllTablesUnderSpecificDatabase( + final PartialPath databasePath) throws MetadataException { + final IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(databasePath).getAsMNode(); + return databaseNode.getChildren().values().stream() + .filter(ConfigTableNode.class::isInstance) + .map( + child -> + new Pair<>( + ((ConfigTableNode) child).getTable(), ((ConfigTableNode) child).getStatus())) + .collect(Collectors.toList()); + } + public Map getSpecificTablesUnderSpecificDatabase( final PartialPath databasePath, final Set tables) throws MetadataException { final IConfigMNode databaseNode = getDatabaseNodeByDatabasePath(databasePath).getAsMNode(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java index 41fb3bfe4693..f6dad2a87dd4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.schema.node.role.IDatabaseMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeContainer; import org.apache.iotdb.commons.schema.node.visitor.MNodeVisitor; +import org.apache.iotdb.commons.schema.table.TableNodeStatus; import org.apache.iotdb.commons.schema.table.TsTable; import org.apache.iotdb.confignode.persistence.schema.mnode.IConfigMNode; import org.apache.iotdb.confignode.persistence.schema.mnode.container.ConfigMNodeContainer; diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/info/ConfigTableInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/info/ConfigTableInfo.java index 73d1cd2928e2..e9a92010a5bf 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/info/ConfigTableInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/info/ConfigTableInfo.java @@ -19,8 +19,8 @@ package org.apache.iotdb.confignode.persistence.schema.mnode.info; +import org.apache.iotdb.commons.schema.table.TableNodeStatus; import org.apache.iotdb.commons.schema.table.TsTable; -import org.apache.iotdb.confignode.persistence.schema.mnode.impl.TableNodeStatus; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.BasicMNodeInfo; public class ConfigTableInfo extends BasicMNodeInfo { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index 1580ed87136f..f2566696ba05 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -1304,8 +1304,8 @@ public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) { } @Override - public TShowTableResp showTables(final String database) { - return configManager.showTables(database); + public TShowTableResp showTables(final String database, final boolean isDetails) { + return configManager.showTables(database, isDetails); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index fe9fdb734124..8487af0dcc94 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -1275,9 +1275,10 @@ public TSStatus alterOrDropTable(final TAlterOrDropTableReq req) throws TExcepti } @Override - public TShowTableResp showTables(final String database) throws TException { + public TShowTableResp showTables(final String database, final boolean isDetails) + throws TException { return executeRemoteCallWithRetry( - () -> client.showTables(database), resp -> !updateConfigNodeLeader(resp.status)); + () -> client.showTables(database, isDetails), resp -> !updateConfigNodeLeader(resp.status)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index a81bab5c5844..ca77a77599ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -3121,7 +3121,7 @@ public SettableFuture showTables( try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - final TShowTableResp resp = configNodeClient.showTables(database); + final TShowTableResp resp = configNodeClient.showTables(database, isDetails); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != resp.getStatus().getCode()) { LOGGER.warn( "Failed to show tables in database {} in config node, status is {}.", @@ -3135,7 +3135,11 @@ public SettableFuture showTables( resp.getStatus().code)); return future; } - ShowTablesTask.buildTsBlock(resp.getTableInfoList(), future); + if (isDetails) { + + } else { + ShowTablesTask.buildTsBlock(resp.getTableInfoList(), future); + } } catch (final Exception e) { future.setException(e); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/TableNodeStatus.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TableNodeStatus.java similarity index 84% rename from iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/TableNodeStatus.java rename to iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TableNodeStatus.java index a3d8e14cd60d..79fbdab24d6b 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/TableNodeStatus.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/TableNodeStatus.java @@ -17,7 +17,7 @@ * under the License. */ -package org.apache.iotdb.confignode.persistence.schema.mnode.impl; +package org.apache.iotdb.commons.schema.table; import org.apache.tsfile.utils.ReadWriteIOUtils; @@ -34,15 +34,15 @@ public enum TableNodeStatus { private final byte status; - private TableNodeStatus(byte status) { + TableNodeStatus(byte status) { this.status = status; } - public void serialize(OutputStream outputStream) throws IOException { + public void serialize(final OutputStream outputStream) throws IOException { ReadWriteIOUtils.write(status, outputStream); } - public static TableNodeStatus deserialize(InputStream inputStream) throws IOException { + public static TableNodeStatus deserialize(final InputStream inputStream) throws IOException { byte status = ReadWriteIOUtils.readByte(inputStream); switch (status) { case 0: diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index f95850ff849f..ffe7859c7bb4 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -1053,6 +1053,7 @@ struct TTableInfo { 1: required string tableName // TTL is stored as string in table props 2: required string TTL + 3: optional i32 state } service IConfigNodeRPCService { @@ -1789,7 +1790,7 @@ service IConfigNodeRPCService { common.TSStatus alterOrDropTable(TAlterOrDropTableReq req) - TShowTableResp showTables(string database) + TShowTableResp showTables(string database, bool isDetails) TFetchTableResp fetchTables(map> fetchTableMap) } From 02257c00bbd89ec721eb17842590ad87d5ea3059 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:45:32 +0800 Subject: [PATCH 085/109] Update ConfigTableNode.java --- .../schema/mnode/impl/ConfigTableNode.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java index f6dad2a87dd4..337f136919ca 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigTableNode.java @@ -42,7 +42,7 @@ public class ConfigTableNode implements IConfigMNode { private ConfigTableInfo tableNodeInfo; - public ConfigTableNode(IConfigMNode parent, String name) { + public ConfigTableNode(final IConfigMNode parent, final String name) { this.parent = parent; this.tableNodeInfo = new ConfigTableInfo(name); } @@ -51,7 +51,7 @@ public TsTable getTable() { return tableNodeInfo.getTable(); } - public void setTable(TsTable table) { + public void setTable(final TsTable table) { tableNodeInfo.setTable(table); } @@ -59,7 +59,7 @@ public TableNodeStatus getStatus() { return tableNodeInfo.getStatus(); } - public void setStatus(TableNodeStatus status) { + public void setStatus(final TableNodeStatus status) { tableNodeInfo.setStatus(status); } @@ -69,7 +69,7 @@ public String getName() { } @Override - public void setName(String name) { + public void setName(final String name) { tableNodeInfo.setName(name); } @@ -79,7 +79,7 @@ public IConfigMNode getParent() { } @Override - public void setParent(IConfigMNode parent) { + public void setParent(final IConfigMNode parent) { this.parent = parent; } @@ -92,7 +92,7 @@ public String getFullPath() { } String concatFullPath() { - StringBuilder builder = new StringBuilder(getName()); + final StringBuilder builder = new StringBuilder(getName()); IConfigMNode curr = this; while (curr.getParent() != null) { curr = curr.getParent(); @@ -102,13 +102,13 @@ String concatFullPath() { } @Override - public void setFullPath(String fullPath) { + public void setFullPath(final String fullPath) { this.fullPath = fullPath; } @Override public PartialPath getPartialPath() { - List detachedPath = new ArrayList<>(); + final List detachedPath = new ArrayList<>(); IConfigMNode temp = this; detachedPath.add(temp.getName()); while (temp.getParent() != null) { @@ -124,7 +124,7 @@ public MNodeType getMNodeType() { } @Override - public R accept(MNodeVisitor visitor, C context) { + public R accept(final MNodeVisitor visitor, final C context) { return visitor.visitBasicMNode(this, context); } @@ -140,28 +140,28 @@ public IConfigMNode getAsMNode() { /** check whether the MNode has a child with the name */ @Override - public boolean hasChild(String name) { + public boolean hasChild(final String name) { return false; } /** get the child with the name */ @Override - public IConfigMNode getChild(String name) { + public IConfigMNode getChild(final String name) { return null; } @Override - public IConfigMNode addChild(String name, IConfigMNode child) { + public IConfigMNode addChild(final String name, final IConfigMNode child) { return null; } @Override - public IConfigMNode addChild(IConfigMNode child) { + public IConfigMNode addChild(final IConfigMNode child) { return null; } @Override - public IConfigMNode deleteChild(String name) { + public IConfigMNode deleteChild(final String name) { return null; } @@ -171,7 +171,7 @@ public IMNodeContainer getChildren() { } @Override - public void setChildren(IMNodeContainer children) { + public void setChildren(final IMNodeContainer children) { // do nothing } @@ -191,7 +191,7 @@ public IDatabaseMNode getAsDatabaseMNode() { } @Override - public void setSchemaTemplateId(int id) { + public void setSchemaTemplateId(final int id) { throw new UnsupportedOperationException(); } From ce1eb3f55ae72713eda0113d6612c0f6cfefb6d8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:47:51 +0800 Subject: [PATCH 086/109] may complete --- .../executor/ClusterConfigTaskExecutor.java | 3 +- .../relational/ShowTablesDetailsTask.java | 47 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index ca77a77599ce..a34c820930ac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -170,6 +170,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.model.ShowModelsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.DescribeTableTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowDBTask; +import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowTablesDetailsTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational.ShowTablesTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.ShowNodesInSchemaTemplateTask; import org.apache.iotdb.db.queryengine.plan.execution.config.metadata.template.ShowPathSetTemplateTask; @@ -3136,7 +3137,7 @@ public SettableFuture showTables( return future; } if (isDetails) { - + ShowTablesDetailsTask.buildTsBlock(resp.getTableInfoList(), future); } else { ShowTablesTask.buildTsBlock(resp.getTableInfoList(), future); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java index d1bbd2a5486a..22521fabcef6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java @@ -19,11 +19,26 @@ package org.apache.iotdb.db.queryengine.plan.execution.config.metadata.relational; +import org.apache.iotdb.commons.schema.table.TableNodeStatus; +import org.apache.iotdb.confignode.rpc.thrift.TTableInfo; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeader; +import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant; +import org.apache.iotdb.db.queryengine.common.header.DatasetHeader; +import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory; import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult; import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask; import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor; +import org.apache.iotdb.rpc.TSStatusCode; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.block.TsBlockBuilder; +import org.apache.tsfile.utils.Binary; + +import java.util.List; +import java.util.stream.Collectors; public class ShowTablesDetailsTask implements IConfigTask { @@ -38,4 +53,36 @@ public ListenableFuture execute(final IConfigTaskExecutor conf throws InterruptedException { return configTaskExecutor.showTables(database, true); } + + public static void buildTsBlock( + final List tableInfoList, final SettableFuture future) { + final List outputDataTypes = + ColumnHeaderConstant.showTablesDetailsColumnHeaders.stream() + .map(ColumnHeader::getColumnType) + .collect(Collectors.toList()); + + final TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes); + + tableInfoList.forEach( + tableInfo -> { + builder.getTimeColumnBuilder().writeLong(0L); + + builder + .getColumnBuilder(0) + .writeBinary(new Binary(tableInfo.getTableName(), TSFileConfig.STRING_CHARSET)); + builder + .getColumnBuilder(1) + .writeBinary(new Binary(tableInfo.getTTL(), TSFileConfig.STRING_CHARSET)); + builder + .getColumnBuilder(2) + .writeBinary( + new Binary( + TableNodeStatus.values()[tableInfo.getState()].toString(), + TSFileConfig.STRING_CHARSET)); + builder.declarePosition(); + }); + + final DatasetHeader datasetHeader = DatasetHeaderFactory.getShowTablesHeader(); + future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); + } } From caf3db0f5b81a3c5f725eaa547c4b2c3cf5f3dc8 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:54:49 +0800 Subject: [PATCH 087/109] Final --- .../db/queryengine/common/header/DatasetHeaderFactory.java | 4 ++++ .../config/metadata/relational/ShowTablesDetailsTask.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java index b29ebb3a132f..9dbbe22b05e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/header/DatasetHeaderFactory.java @@ -228,4 +228,8 @@ public static DatasetHeader getDescribeTableHeader() { public static DatasetHeader getShowTablesHeader() { return new DatasetHeader(ColumnHeaderConstant.showTablesColumnHeaders, true); } + + public static DatasetHeader getShowTablesDetailsHeader() { + return new DatasetHeader(ColumnHeaderConstant.showTablesDetailsColumnHeaders, true); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java index 22521fabcef6..1ecd87d48753 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/ShowTablesDetailsTask.java @@ -82,7 +82,7 @@ public static void buildTsBlock( builder.declarePosition(); }); - final DatasetHeader datasetHeader = DatasetHeaderFactory.getShowTablesHeader(); + final DatasetHeader datasetHeader = DatasetHeaderFactory.getShowTablesDetailsHeader(); future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader)); } } From eb6ee889b3065df10e147046bfe8f35647735ce9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:01:14 +0800 Subject: [PATCH 088/109] Update DropTableTask.java --- .../execution/config/metadata/relational/DropTableTask.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java index fb5f610a36f7..34c59a745e8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/DropTableTask.java @@ -37,6 +37,6 @@ public DropTableTask( @Override public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { - return null; + return configTaskExecutor.dropTable(database, tableName, queryId, tableIfExists); } } From 812a8f3ef6528912cdf2a18e063b6641f7bc99a3 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:06:33 +0800 Subject: [PATCH 089/109] Update AlterOrDropTableOperationType.java --- .../commons/schema/table/AlterOrDropTableOperationType.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterOrDropTableOperationType.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterOrDropTableOperationType.java index 3a742bb20aa5..bdae8196a5d1 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterOrDropTableOperationType.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/table/AlterOrDropTableOperationType.java @@ -49,6 +49,8 @@ public static AlterOrDropTableOperationType getType(final byte value) { return DROP_COLUMN; case 4: return RENAME_TABLE; + case 5: + return DROP_TABLE; default: throw new IllegalArgumentException(); } From eb69454eb4edd4eb0a59bf39c1bea55f71d74a4d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:08:45 +0800 Subject: [PATCH 090/109] Update ProcedureFactory.java --- .../iotdb/confignode/procedure/store/ProcedureFactory.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java index 9cf5011d5e41..402d5b6ed28f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureFactory.java @@ -363,6 +363,12 @@ public static ProcedureType getProcedureType(Procedure procedure) { return ProcedureType.ADD_TABLE_COLUMN_PROCEDURE; } else if (procedure instanceof SetTablePropertiesProcedure) { return ProcedureType.SET_TABLE_PROPERTIES_PROCEDURE; + } else if (procedure instanceof RenameTableColumnProcedure) { + return ProcedureType.RENAME_TABLE_COLUMN_PROCEDURE; + } else if (procedure instanceof DropTableColumnProcedure) { + return ProcedureType.DROP_TABLE_COLUMN_PROCEDURE; + } else if (procedure instanceof DropTableProcedure) { + return ProcedureType.DROP_TABLE_PROCEDURE; } else if (procedure instanceof CreatePipePluginProcedure) { return ProcedureType.CREATE_PIPE_PLUGIN_PROCEDURE; } else if (procedure instanceof DropPipePluginProcedure) { From 15f45aa24e776a62713952a794b2344efd9d0b13 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:11:28 +0800 Subject: [PATCH 091/109] Update DropTableProcedure.java --- .../impl/schema/table/DropTableProcedure.java | 48 +++++++++---------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 39c0b1af5333..8fa69090a3e4 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -173,21 +173,19 @@ private void deleteData(final ConfigNodeProcedureEnv env) { final Map relatedDataRegionGroup = env.getConfigManager().getRelatedDataRegionGroup(patternTree); - if (relatedDataRegionGroup.isEmpty()) { - return; + if (!relatedDataRegionGroup.isEmpty()) { + new DropTableRegionTaskExecutor<>( + "delete data for drop table", + env, + relatedDataRegionGroup, + true, + CnToDnAsyncRequestType.DELETE_DATA_FOR_DROP_TABLE, + ((dataNodeLocation, consensusGroupIdList) -> + new TDeleteDataOrDevicesForDropTableReq( + new ArrayList<>(consensusGroupIdList), tableName))) + .execute(); } - new DropTableRegionTaskExecutor<>( - "delete data for drop table", - env, - relatedDataRegionGroup, - true, - CnToDnAsyncRequestType.DELETE_DATA_FOR_DROP_TABLE, - ((dataNodeLocation, consensusGroupIdList) -> - new TDeleteDataOrDevicesForDropTableReq( - new ArrayList<>(consensusGroupIdList), tableName))) - .execute(); - setNextState(DropTableState.DELETE_DEVICES); } @@ -211,21 +209,19 @@ private void deleteSchema(final ConfigNodeProcedureEnv env) { final Map relatedSchemaRegionGroup = env.getConfigManager().getRelatedSchemaRegionGroup(patternTree); - if (relatedSchemaRegionGroup.isEmpty()) { - return; + if (!relatedSchemaRegionGroup.isEmpty()) { + new DropTableRegionTaskExecutor<>( + "delete devices for drop table", + env, + relatedSchemaRegionGroup, + true, + CnToDnAsyncRequestType.DELETE_DEVICES_FOR_DROP_TABLE, + ((dataNodeLocation, consensusGroupIdList) -> + new TDeleteDataOrDevicesForDropTableReq( + new ArrayList<>(consensusGroupIdList), tableName))) + .execute(); } - new DropTableRegionTaskExecutor<>( - "delete devices for drop table", - env, - relatedSchemaRegionGroup, - true, - CnToDnAsyncRequestType.DELETE_DEVICES_FOR_DROP_TABLE, - ((dataNodeLocation, consensusGroupIdList) -> - new TDeleteDataOrDevicesForDropTableReq( - new ArrayList<>(consensusGroupIdList), tableName))) - .execute(); - setNextState(DropTableState.DROP_TABLE); } From 00414ee1fa34727efa0c0016e88023e45dc5b8b0 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:13:25 +0800 Subject: [PATCH 092/109] Update ConfigMTree.java --- .../apache/iotdb/confignode/persistence/schema/ConfigMTree.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java index b606bf399268..df93713239ac 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ConfigMTree.java @@ -681,7 +681,7 @@ public void preDeleteTable(final PartialPath database, final String tableName) database.getFullPath().substring(ROOT.length() + 1), tableName); } final ConfigTableNode tableNode = (ConfigTableNode) databaseNode.getChild(tableName); - if (!tableNode.getStatus().equals(TableNodeStatus.USING)) { + if (tableNode.getStatus().equals(TableNodeStatus.PRE_CREATE)) { throw new IllegalStateException(); } tableNode.setStatus(TableNodeStatus.PRE_DELETE); From 122dea7aae6354c05f6f57d6901d589662d5a9d2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:26:27 +0800 Subject: [PATCH 093/109] Update MNodeCollector.java --- .../mtree/traverser/collector/MNodeCollector.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/collector/MNodeCollector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/collector/MNodeCollector.java index fdbb2419c78a..8e218aa928a1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/collector/MNodeCollector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/collector/MNodeCollector.java @@ -35,18 +35,18 @@ public abstract class MNodeCollector> extends MNodeTraverser { protected MNodeCollector( - N startNode, - PartialPath path, - IMTreeStore store, - boolean isPrefixMatch, - PathPatternTree scope) + final N startNode, + final PartialPath path, + final IMTreeStore store, + final boolean isPrefixMatch, + final PathPatternTree scope) throws MetadataException { super(startNode, path, store, isPrefixMatch, scope); } - protected final R transferToResult(N node) { + protected final R transferToResult(final N node) { return collectMNode(node); } - protected abstract R collectMNode(N node); + protected abstract R collectMNode(final N node); } From 24f18fe5e44db1632a2507345b9c63504b2e830c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:49:24 +0800 Subject: [PATCH 094/109] Partial --- .../attribute/DeviceAttributeStore.java | 17 +++++++-- .../attribute/IDeviceAttributeStore.java | 2 ++ .../impl/mem/MTreeBelowSGMemoryImpl.java | 36 ++++++++++++++++--- .../mtree/impl/mem/MemMTreeStore.java | 2 +- .../traverser/updater/EntityUpdater.java | 1 + .../iotdb/commons/schema/MemUsageUtil.java | 2 +- 6 files changed, 50 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java index 7ac2618e114d..6fa60e9c224b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -27,6 +27,7 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,8 +49,11 @@ public class DeviceAttributeStore implements IDeviceAttributeStore { private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeStore.class); + private static final long MAP_SIZE_WITH_REF = + RamUsageEstimator.shallowSizeOfInstance(HashMap.class) + + RamUsageEstimator.NUM_BYTES_OBJECT_REF; - public List> deviceAttributeList = new ArrayList<>(); + private final List> deviceAttributeList = new ArrayList<>(); private final MemSchemaRegionStatistics regionStatistics; @@ -59,7 +63,7 @@ public DeviceAttributeStore(final MemSchemaRegionStatistics regionStatistics) { @Override public void clear() { - deviceAttributeList = new ArrayList<>(); + deviceAttributeList.clear(); } @Override @@ -127,7 +131,7 @@ public void loadFromSnapshot(final File snapshotDir, final String sgSchemaDirPat @Override public synchronized int createAttribute(final List nameList, final Object[] valueList) { // todo implement storage for device of diverse data types - long memUsage = 0L; + long memUsage = MAP_SIZE_WITH_REF; final Map attributeMap = new HashMap<>(); String value; for (int i = 0; i < nameList.size(); i++) { @@ -184,6 +188,13 @@ public Map alterAttribute( return updateMap; } + @Override + public Map removeAttribute(final int pointer) { + + deviceAttributeList.set(pointer, null); + return null; + } + @Override public String getAttribute(final int pointer, final String name) { return deviceAttributeList.get(pointer).get(name); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java index e87ee18b5d46..f17d36db6050 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/IDeviceAttributeStore.java @@ -38,5 +38,7 @@ public interface IDeviceAttributeStore { Map alterAttribute( final int pointer, final List nameList, final Object[] valueList); + Map removeAttribute(final int pointer); + String getAttribute(final int pointer, final String name); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 054860e854ad..0ce07c821c66 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -98,6 +98,7 @@ import java.util.NoSuchElementException; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiFunction; import java.util.function.Consumer; import java.util.function.Function; @@ -1585,12 +1586,37 @@ protected void updateEntity(final IDeviceMNode node) { public void renameTableAttribute() {} - public boolean deleteTableDevice(final String tableName) { - if (store.hasChild(storageGroupMNode, tableName)) { - store.deleteChild(storageGroupMNode, tableName); - return true; + public boolean deleteTableDevice(final String tableName) throws MetadataException { + if (!store.hasChild(storageGroupMNode, tableName)) { + return false; } - return false; + final AtomicInteger memoryReleased = new AtomicInteger(0); + try (final MNodeCollector collector = + new MNodeCollector( + this.rootNode, + new PartialPath(storageGroupMNode.getFullPath()), + this.store, + true, + SchemaConstant.ALL_MATCH_SCOPE) { + @Override + protected boolean acceptInternalMatchedNode(final IMemMNode node) { + return true; + } + + @Override + protected Void collectMNode(final IMemMNode node) { + if (node.isDevice()) { + deviceProcess.accept(node.getAsDeviceMNode()); + } + memoryReleased.addAndGet(node.estimateSize()); + return null; + } + }) { + collector.traverse(); + } + storageGroupMNode.deleteChild(tableName); + store.releaseMemory(memoryReleased.get()); + return true; } // endregion diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java index 470cacd8d9f5..be95463a4915 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java @@ -239,7 +239,7 @@ private void requestMemory(int size) { } } - private void releaseMemory(int size) { + public void releaseMemory(int size) { if (regionStatistics != null) { regionStatistics.releaseMemory(size); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/updater/EntityUpdater.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/updater/EntityUpdater.java index d11cdb291439..4641bffcf93f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/updater/EntityUpdater.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/updater/EntityUpdater.java @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.updater; import org.apache.iotdb.commons.exception.MetadataException; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/MemUsageUtil.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/MemUsageUtil.java index afc25156dfbf..a4a146390ba8 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/MemUsageUtil.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/MemUsageUtil.java @@ -41,7 +41,7 @@ public static long computeStringMemUsage(String value) { * * */ - public static long computeKVMemUsageInMap(String key, String value) { + public static long computeKVMemUsageInMap(final String key, final String value) { return 40L + estimateStringSize(key) + estimateStringSize(value); } From ebeaa0d0edd6e643960163bc49082bb446088b1a Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:54:56 +0800 Subject: [PATCH 095/109] Memory calculation --- .../attribute/DeviceAttributeStore.java | 3 ++- .../update/UpdateDetailContainer.java | 22 ++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java index 6fa60e9c224b..309cefc42391 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.attribute.update.UpdateDetailContainer; import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.utils.Binary; @@ -190,7 +191,7 @@ public Map alterAttribute( @Override public Map removeAttribute(final int pointer) { - + releaseMemory(UpdateDetailContainer.sizeOfMapEntries(deviceAttributeList.get(pointer))); deviceAttributeList.set(pointer, null); return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index b5b018ccec18..a030c4e83b12 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -264,15 +264,7 @@ public Pair updateSelfByCommitContainer(final UpdateContainer com .mapToLong( entry -> sizeOfList(entry.getKey()) - + entry.getValue().size() - * RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY - + entry.getValue().entrySet().stream() - .mapToLong( - attributeKV -> - RamUsageEstimator.sizeOf(attributeKV.getKey()) - + RamUsageEstimator.sizeOf( - attributeKV.getValue())) - .reduce(0, Long::sum)) + + sizeOfMapEntries(entry.getValue())) .reduce(0, Long::sum)); } }); @@ -280,10 +272,20 @@ public Pair updateSelfByCommitContainer(final UpdateContainer com return new Pair<>(result.get(), updateMap.isEmpty()); } - private static long sizeOfList(final List input) { + private static long sizeOfList(final @Nonnull List input) { return input.stream().map(RamUsageEstimator::sizeOf).reduce(LIST_SIZE, Long::sum); } + public static long sizeOfMapEntries(final @Nonnull Map map) { + return map.size() * RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + + map.entrySet().stream() + .mapToLong( + attributeKV -> + RamUsageEstimator.sizeOf(attributeKV.getKey()) + + RamUsageEstimator.sizeOf(attributeKV.getValue())) + .reduce(0, Long::sum); + } + @Override public void serialize(final OutputStream outputStream) throws IOException { ReadWriteIOUtils.write((byte) 1, outputStream); From a5b27be2fec43321ef0f8ba09607adf7d8176dea Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 16:57:11 +0800 Subject: [PATCH 096/109] May add calculation --- .../schemaregion/attribute/DeviceAttributeStore.java | 9 ++++----- .../attribute/update/UpdateDetailContainer.java | 6 +++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java index 309cefc42391..b722bc892b41 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/DeviceAttributeStore.java @@ -50,9 +50,7 @@ public class DeviceAttributeStore implements IDeviceAttributeStore { private static final Logger logger = LoggerFactory.getLogger(DeviceAttributeStore.class); - private static final long MAP_SIZE_WITH_REF = - RamUsageEstimator.shallowSizeOfInstance(HashMap.class) - + RamUsageEstimator.NUM_BYTES_OBJECT_REF; + private static final long MAP_SIZE = RamUsageEstimator.shallowSizeOfInstance(HashMap.class); private final List> deviceAttributeList = new ArrayList<>(); @@ -132,7 +130,7 @@ public void loadFromSnapshot(final File snapshotDir, final String sgSchemaDirPat @Override public synchronized int createAttribute(final List nameList, final Object[] valueList) { // todo implement storage for device of diverse data types - long memUsage = MAP_SIZE_WITH_REF; + long memUsage = MAP_SIZE + RamUsageEstimator.NUM_BYTES_OBJECT_REF; final Map attributeMap = new HashMap<>(); String value; for (int i = 0; i < nameList.size(); i++) { @@ -191,7 +189,8 @@ public Map alterAttribute( @Override public Map removeAttribute(final int pointer) { - releaseMemory(UpdateDetailContainer.sizeOfMapEntries(deviceAttributeList.get(pointer))); + releaseMemory( + MAP_SIZE + UpdateDetailContainer.sizeOfMapEntries(deviceAttributeList.get(pointer))); deviceAttributeList.set(pointer, null); return null; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java index a030c4e83b12..de972baa187b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/attribute/update/UpdateDetailContainer.java @@ -280,9 +280,9 @@ public static long sizeOfMapEntries(final @Nonnull Map map) { return map.size() * RamUsageEstimator.HASHTABLE_RAM_BYTES_PER_ENTRY + map.entrySet().stream() .mapToLong( - attributeKV -> - RamUsageEstimator.sizeOf(attributeKV.getKey()) - + RamUsageEstimator.sizeOf(attributeKV.getValue())) + innerEntry -> + RamUsageEstimator.sizeOf(innerEntry.getKey()) + + RamUsageEstimator.sizeOf(innerEntry.getValue())) .reduce(0, Long::sum); } From c58352f7ac1b80cb161fc5efc4d742aace04c923 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:18:55 +0800 Subject: [PATCH 097/109] Revert --- .../mnode/impl/ConfigBasicInternalMNode.java | 6 ------ .../impl/SchemaRegionMemoryImpl.java | 3 ++- .../mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 7 +++++-- .../mtree/impl/mem/MemMTreeStore.java | 2 +- .../impl/mem/mnode/impl/BasicInternalMNode.java | 5 ----- .../mnode/impl/CachedBasicInternalMNode.java | 5 ----- .../apache/iotdb/commons/schema/node/IMNode.java | 4 ---- .../commons/schema/node/role/IInternalMNode.java | 16 ---------------- 8 files changed, 8 insertions(+), 40 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigBasicInternalMNode.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigBasicInternalMNode.java index eb99e1c8b52c..b61d5570a608 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigBasicInternalMNode.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/mnode/impl/ConfigBasicInternalMNode.java @@ -19,7 +19,6 @@ package org.apache.iotdb.confignode.persistence.schema.mnode.impl; import org.apache.iotdb.commons.schema.node.MNodeType; -import org.apache.iotdb.commons.schema.node.role.IInternalMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeContainer; import org.apache.iotdb.confignode.persistence.schema.mnode.IConfigMNode; import org.apache.iotdb.confignode.persistence.schema.mnode.basic.ConfigBasicMNode; @@ -145,11 +144,6 @@ public int estimateSize() { return 8 + 80 + super.estimateSize(); } - @Override - public int estimateTreeSize() { - return estimateSize() + IInternalMNode.getChildrenTreeSize(children); - } - @Override public MNodeType getMNodeType() { return MNodeType.SG_INTERNAL; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index 8fe129e2ed1d..6abf796c03b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -1521,7 +1521,8 @@ private DeviceAttributeUpdater constructDevicePredicateUpdater( @Override public void deleteTableDevice(final DeleteTableDeviceNode deleteTableDeviceNode) throws MetadataException { - if (mtree.deleteTableDevice(deleteTableDeviceNode.getTableName())) { + if (mtree.deleteTableDevice( + deleteTableDeviceNode.getTableName(), deviceAttributeStore::removeAttribute)) { writeToMLog(deleteTableDeviceNode); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 0ce07c821c66..3f4075441ec4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -1586,7 +1586,8 @@ protected void updateEntity(final IDeviceMNode node) { public void renameTableAttribute() {} - public boolean deleteTableDevice(final String tableName) throws MetadataException { + public boolean deleteTableDevice(final String tableName, final IntConsumer attributeDeleter) + throws MetadataException { if (!store.hasChild(storageGroupMNode, tableName)) { return false; } @@ -1606,7 +1607,9 @@ protected boolean acceptInternalMatchedNode(final IMemMNode node) { @Override protected Void collectMNode(final IMemMNode node) { if (node.isDevice()) { - deviceProcess.accept(node.getAsDeviceMNode()); + attributeDeleter.accept( + ((TableDeviceInfo) node.getAsDeviceMNode().getDeviceInfo()) + .getAttributePointer()); } memoryReleased.addAndGet(node.estimateSize()); return null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java index be95463a4915..b8f993fa93dc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MemMTreeStore.java @@ -133,7 +133,7 @@ public IMemMNode addChild(final IMemMNode parent, final String childName, final @Override public void deleteChild(final IMemMNode parent, final String childName) { - releaseMemory(parent.deleteChild(childName).estimateTreeSize()); + releaseMemory(parent.deleteChild(childName).estimateSize()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/BasicInternalMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/BasicInternalMNode.java index 15801ef35618..904a0a5b5ba1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/BasicInternalMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/BasicInternalMNode.java @@ -159,11 +159,6 @@ public int estimateSize() { return 8 + 80 + super.estimateSize() + 8 + (deviceInfo == null ? 0 : deviceInfo.estimateSize()); } - @Override - public int estimateTreeSize() { - return estimateSize() + IInternalMNode.getChildrenTreeSize(children); - } - @Override public MNodeType getMNodeType() { return deviceInfo == null ? MNodeType.INTERNAL : MNodeType.DEVICE; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedBasicInternalMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedBasicInternalMNode.java index 7b29fd8d0c64..1b4e67e623df 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedBasicInternalMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedBasicInternalMNode.java @@ -176,11 +176,6 @@ public int estimateSize() { + (deviceInfo == null ? 0 : deviceInfo.estimateSize()); } - @Override - public int estimateTreeSize() { - return estimateSize() + IInternalMNode.getChildrenTreeSize(children); - } - @Override public boolean isDevice() { return getDeviceInfo() != null; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java index 87ea5b00407a..5b26436932cb 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/IMNode.java @@ -81,9 +81,5 @@ public interface IMNode> extends ITreeNode { int estimateSize(); - default int estimateTreeSize() { - return estimateSize(); - } - N getAsMNode(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/role/IInternalMNode.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/role/IInternalMNode.java index 3cdfdfd5231a..ed69923b86d0 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/role/IInternalMNode.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/node/role/IInternalMNode.java @@ -22,11 +22,6 @@ import org.apache.iotdb.commons.schema.node.IMNode; import org.apache.iotdb.commons.schema.node.common.DeviceMNodeWrapper; import org.apache.iotdb.commons.schema.node.info.IDeviceInfo; -import org.apache.iotdb.commons.schema.node.utils.IMNodeContainer; - -import org.apache.tsfile.utils.RamUsageEstimator; - -import java.util.Objects; public interface IInternalMNode> extends IMNode { @@ -57,15 +52,4 @@ default IDeviceMNode getAsDeviceMNode() { throw new UnsupportedOperationException("Wrong node type"); } } - - static int getChildrenTreeSize(final IMNodeContainer> children) { - return Objects.nonNull(children) - ? children.entrySet().stream() - .map( - entry -> - (int) RamUsageEstimator.sizeOf(entry.getKey()) - + entry.getValue().estimateTreeSize()) - .reduce(0, Integer::sum) - : 0; - } } From 394a86a04a57f972f8c48d42d5fcb68fa23085bb Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:45:09 +0800 Subject: [PATCH 098/109] Update MTreeBelowSGMemoryImpl.java --- .../schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 3f4075441ec4..084a9c54065b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -1595,7 +1595,7 @@ public boolean deleteTableDevice(final String tableName, final IntConsumer attri try (final MNodeCollector collector = new MNodeCollector( this.rootNode, - new PartialPath(storageGroupMNode.getFullPath()), + storageGroupMNode.getPartialPath().concatNode(tableName), this.store, true, SchemaConstant.ALL_MATCH_SCOPE) { @@ -1606,6 +1606,7 @@ protected boolean acceptInternalMatchedNode(final IMemMNode node) { @Override protected Void collectMNode(final IMemMNode node) { + System.out.println(node.getName()); if (node.isDevice()) { attributeDeleter.accept( ((TableDeviceInfo) node.getAsDeviceMNode().getDeviceInfo()) From 240d1444f81b47a4bb57e18747fb4ba6bd9e2d6c Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:48:52 +0800 Subject: [PATCH 099/109] fix --- .../procedure/impl/schema/table/CreateTableProcedure.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index b48072996eb1..daf6b0a5d788 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -191,6 +191,7 @@ private void validateTimeSeriesExistence(final ConfigNodeProcedureEnv env) { env.getConfigManager().getRelatedSchemaRegionGroup(patternTree); if (relatedSchemaRegionGroup.isEmpty()) { + setNextState(CreateTableState.COMMIT_CREATE); return; } From 832ca9abc767792a9362f14596aa9cf721c726a9 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:04:03 +0800 Subject: [PATCH 100/109] Fix --- .../procedure/impl/schema/table/CreateTableProcedure.java | 2 +- .../procedure/impl/schema/table/DropTableProcedure.java | 4 ++-- .../relational/planner/node/schema/DeleteTableDeviceNode.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java index daf6b0a5d788..b4f395d9a299 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/CreateTableProcedure.java @@ -178,7 +178,7 @@ private void validateTimeSeriesExistence(final ConfigNodeProcedureEnv env) { final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); final PartialPath path; try { - path = new PartialPath(new String[] {ROOT, database, table.getTableName()}); + path = new PartialPath(new String[] {ROOT, database.substring(5), table.getTableName()}); patternTree.appendPathPattern(path); patternTree.appendPathPattern(path.concatAsMeasurementPath(MULTI_LEVEL_PATH_WILDCARD)); patternTree.serialize(dataOutputStream); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 8fa69090a3e4..78435583501d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -162,7 +162,7 @@ private void deleteData(final ConfigNodeProcedureEnv env) { final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); final PartialPath path; try { - path = new PartialPath(new String[] {ROOT, database, tableName}); + path = new PartialPath(new String[] {ROOT, database.substring(5), tableName}); patternTree.appendPathPattern(path); patternTree.appendPathPattern(path.concatAsMeasurementPath(MULTI_LEVEL_PATH_WILDCARD)); patternTree.serialize(dataOutputStream); @@ -196,7 +196,7 @@ private void deleteSchema(final ConfigNodeProcedureEnv env) { final DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream); final PartialPath path; try { - path = new PartialPath(new String[] {ROOT, database, tableName}); + path = new PartialPath(new String[] {ROOT, database.substring(5), tableName}); patternTree.appendPathPattern(path); patternTree.appendPathPattern(path.concatAsMeasurementPath(MULTI_LEVEL_PATH_WILDCARD)); patternTree.serialize(dataOutputStream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java index e269a4ea5a1d..5cfccb784e63 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/schema/DeleteTableDeviceNode.java @@ -55,7 +55,7 @@ public List getChildren() { } @Override - public void addChild(PlanNode child) { + public void addChild(final PlanNode child) { throw new UnsupportedOperationException(); } @@ -98,7 +98,7 @@ public static DeleteTableDeviceNode deserialize(final ByteBuffer buffer) { } @Override - public R accept(final PlanVisitor visitor, C context) { + public R accept(final PlanVisitor visitor, final C context) { return visitor.visitDeleteTableDevice(this, context); } From 34c7f47971911f1331ab108f089add4af7db5902 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:25:49 +0800 Subject: [PATCH 101/109] Fix --- .../schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java | 5 ++--- .../schemaengine/schemaregion/mtree/traverser/Traverser.java | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index 084a9c54065b..b84e97ec5642 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -1594,8 +1594,8 @@ public boolean deleteTableDevice(final String tableName, final IntConsumer attri final AtomicInteger memoryReleased = new AtomicInteger(0); try (final MNodeCollector collector = new MNodeCollector( - this.rootNode, - storageGroupMNode.getPartialPath().concatNode(tableName), + storageGroupMNode, + new PartialPath(new String[] {storageGroupMNode.getName(), tableName}), this.store, true, SchemaConstant.ALL_MATCH_SCOPE) { @@ -1606,7 +1606,6 @@ protected boolean acceptInternalMatchedNode(final IMemMNode node) { @Override protected Void collectMNode(final IMemMNode node) { - System.out.println(node.getName()); if (node.isDevice()) { attributeDeleter.accept( ((TableDeviceInfo) node.getAsDeviceMNode().getDeviceInfo()) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java index f1ac33d99ed6..e23d8251139a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java @@ -103,7 +103,7 @@ protected Traverser( this.store = store.getWithReentrantReadLock(); initStack(); String[] nodes = path.getNodes(); - if (nodes.length == 0 || !nodes[0].equals(PATH_ROOT)) { + if (nodes.length == 0 || !nodes[0].equals(startNode.getName())) { throw new IllegalPathException( path.getFullPath(), path.getFullPath() + " doesn't start with " + startNode.getName()); } From 6b1298eaae3ceffc5c67dff8767f4e75f9f8466e Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 18:26:52 +0800 Subject: [PATCH 102/109] Update Traverser.java --- .../db/schemaengine/schemaregion/mtree/traverser/Traverser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java index e23d8251139a..fb7294965a9b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java @@ -45,7 +45,6 @@ import java.util.NoSuchElementException; import java.util.Objects; -import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_ROOT; import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; /** From 6b56ffe8302a3aa845c5657fdcdfd8e4ab383965 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 20:01:47 +0800 Subject: [PATCH 103/109] Update IoTDBTableIT.java --- .../relational/it/schema/IoTDBTableIT.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index 400724e66d0f..be0081294e67 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -19,6 +19,7 @@ package org.apache.iotdb.relational.it.schema; +import org.apache.iotdb.db.it.utils.TestUtils; import org.apache.iotdb.it.env.EnvFactory; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.TableClusterIT; @@ -36,6 +37,7 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Statement; +import java.util.Collections; import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.describeTableColumnHeaders; import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.showTablesColumnHeaders; @@ -368,6 +370,22 @@ public void testManageTable() { assertEquals(columnNames.length, cnt); } + statement.execute( + "insert into table2(region_id, plant_id, color, temperature, speed) values(1, 1, 1, 1, 1)"); + statement.execute("drop table2"); + try { + statement.executeQuery("describe table2"); + fail(); + } catch (final SQLException e) { + assertEquals("550: Table 'test2.table2' does not exist.", e.getMessage()); + } + statement.execute( + "create table table2(region_id STRING ID, plant_id STRING ID, color STRING ATTRIBUTE, temperature FLOAT MEASUREMENT, speed DOUBLE MEASUREMENT)"); + TestUtils.assertResultSetEqual( + statement.executeQuery("count devices from table2"), + "count(devices),", + Collections.singleton("0,")); + try { statement.executeQuery("describe test3.table3"); fail(); From b4fcc5f0c89ef435ce19fafd5f7e512d9eb3fd5f Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 20:22:19 +0800 Subject: [PATCH 104/109] Update IoTDBTableIT.java --- .../org/apache/iotdb/relational/it/schema/IoTDBTableIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java index be0081294e67..cc04aa8816dd 100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBTableIT.java @@ -372,7 +372,7 @@ public void testManageTable() { statement.execute( "insert into table2(region_id, plant_id, color, temperature, speed) values(1, 1, 1, 1, 1)"); - statement.execute("drop table2"); + statement.execute("drop table table2"); try { statement.executeQuery("describe table2"); fail(); From 0aac1e60569859e7d93cf206ed579576ea655cae Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 21:55:04 +0800 Subject: [PATCH 105/109] Update TableDeviceSchemaCacheTest.java --- .../cache/TableDeviceSchemaCacheTest.java | 199 ++++++++++-------- 1 file changed, 109 insertions(+), 90 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java index d739d9493bce..942fa11e8cb7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java @@ -37,8 +37,9 @@ import org.apache.tsfile.utils.Pair; import org.apache.tsfile.utils.TsPrimitiveType; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; import java.util.Arrays; @@ -56,116 +57,157 @@ public class TableDeviceSchemaCacheTest { private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); - private long originMemConfig; + private static long originMemConfig; + + private static final String database1 = "sg1"; + private static final String database2 = "sg2"; + private static final String table1 = "t1"; + private static final String table2 = "t2"; + + @BeforeClass + public static void prepareEnvironment() { + final List columnHeaderList = + Arrays.asList( + new ColumnHeader("hebei", TSDataType.STRING), + new ColumnHeader("p_1", TSDataType.STRING), + new ColumnHeader("d_1", TSDataType.STRING)); + final String attributeName = "attr"; + + // Prepare tables + final TsTable testTable1 = new TsTable(table1); + columnHeaderList.forEach( + columnHeader -> + testTable1.addColumnSchema( + new IdColumnSchema(columnHeader.getColumnName(), columnHeader.getColumnType()))); + testTable1.addColumnSchema(new AttributeColumnSchema(attributeName, TSDataType.STRING)); + testTable1.addColumnSchema(new TimeColumnSchema("time", TSDataType.INT64)); + testTable1.addColumnSchema( + new MeasurementColumnSchema( + "s1", TSDataType.BOOLEAN, TSEncoding.RLE, CompressionType.GZIP)); + DataNodeTableCache.getInstance().preUpdateTable(database1, testTable1); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table1); + + DataNodeTableCache.getInstance().preUpdateTable(database2, testTable1); + DataNodeTableCache.getInstance().commitUpdateTable(database2, table1); + + final TsTable testTable2 = new TsTable(table2); + columnHeaderList.forEach( + columnHeader -> + testTable2.addColumnSchema( + new IdColumnSchema(columnHeader.getColumnName(), columnHeader.getColumnType()))); + testTable2.addColumnSchema(new AttributeColumnSchema(attributeName, TSDataType.STRING)); + testTable2.addColumnSchema(new TimeColumnSchema("time", TSDataType.INT64)); + testTable2.addColumnSchema( + new MeasurementColumnSchema( + "s1", TSDataType.BOOLEAN, TSEncoding.RLE, CompressionType.GZIP)); + DataNodeTableCache.getInstance().preUpdateTable(database1, testTable2); + DataNodeTableCache.getInstance().commitUpdateTable(database1, table2); - @Before - public void setup() { originMemConfig = config.getAllocateMemoryForSchemaCache(); config.setAllocateMemoryForSchemaCache(1300L); } + @AfterClass + public static void clearEnvironment() { + DataNodeTableCache.getInstance().invalid(database1); + config.setAllocateMemoryForSchemaCache(originMemConfig); + } + @After public void rollback() { TableDeviceSchemaCache.getInstance().invalidateAll(); - config.setAllocateMemoryForSchemaCache(originMemConfig); } @Test public void testDeviceCache() { final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); - final String database = "db"; - final String table1 = "t1"; - final Map attributeMap = new HashMap<>(); attributeMap.put("type", "new"); attributeMap.put("cycle", "monthly"); cache.putAttributes( - database, + database1, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_0"}), new ConcurrentHashMap<>(attributeMap)); Assert.assertEquals( attributeMap, cache.getDeviceAttribute( - database, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_0"}))); + database1, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_0"}))); Assert.assertNull( cache.getDeviceAttribute( - database, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_1"}))); + database1, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_1"}))); attributeMap.put("type", "old"); cache.putAttributes( - database, + database1, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_1"}), new HashMap<>(attributeMap)); Assert.assertEquals( attributeMap, cache.getDeviceAttribute( - database, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_1"}))); + database1, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_1"}))); attributeMap.put("cycle", "daily"); cache.putAttributes( - database, + database1, convertIdValuesToDeviceID(table1, new String[] {"shandong", "p_1", "d_1"}), new ConcurrentHashMap<>(attributeMap)); Assert.assertNull( cache.getDeviceAttribute( - database, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_0"}))); + database1, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_0"}))); Assert.assertEquals( attributeMap, cache.getDeviceAttribute( - database, convertIdValuesToDeviceID(table1, new String[] {"shandong", "p_1", "d_1"}))); + database1, convertIdValuesToDeviceID(table1, new String[] {"shandong", "p_1", "d_1"}))); final String table2 = "t2"; attributeMap.put("type", "new"); attributeMap.put("cycle", "monthly"); cache.putAttributes( - database, + database1, convertIdValuesToDeviceID(table2, new String[] {"hebei", "p_1", "d_0"}), new ConcurrentHashMap<>(attributeMap)); Assert.assertEquals( attributeMap, cache.getDeviceAttribute( - database, convertIdValuesToDeviceID(table2, new String[] {"hebei", "p_1", "d_0"}))); + database1, convertIdValuesToDeviceID(table2, new String[] {"hebei", "p_1", "d_0"}))); Assert.assertNull( cache.getDeviceAttribute( - database, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_1"}))); + database1, convertIdValuesToDeviceID(table1, new String[] {"hebei", "p_1", "d_1"}))); attributeMap.put("type", "old"); cache.putAttributes( - database, + database1, convertIdValuesToDeviceID(table2, new String[] {"hebei", "p_1", "d_1"}), new ConcurrentHashMap<>(attributeMap)); Assert.assertEquals( attributeMap, cache.getDeviceAttribute( - database, convertIdValuesToDeviceID(table2, new String[] {"hebei", "p_1", "d_1"}))); + database1, convertIdValuesToDeviceID(table2, new String[] {"hebei", "p_1", "d_1"}))); Assert.assertNull( cache.getDeviceAttribute( - database, convertIdValuesToDeviceID(table1, new String[] {"shandong", "p_1", "d_1"}))); + database1, convertIdValuesToDeviceID(table1, new String[] {"shandong", "p_1", "d_1"}))); cache.invalidateAttributes( - database, convertIdValuesToDeviceID(table2, new String[] {"hebei", "p_1", "d_1"})); + database1, convertIdValuesToDeviceID(table2, new String[] {"hebei", "p_1", "d_1"})); Assert.assertNull( cache.getDeviceAttribute( - database, convertIdValuesToDeviceID(table2, new String[] {"hebei", "p_1", "d_1"}))); + database1, convertIdValuesToDeviceID(table2, new String[] {"hebei", "p_1", "d_1"}))); } @Test public void testLastCache() { final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); - final String database = "db"; - final String table1 = "t1"; - final String[] device0 = new String[] {"hebei", "p_1", "d_0"}; // Test get from empty cache Assert.assertNull( - cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s0")); + cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s0")); Assert.assertFalse( cache .getLastRow( - database, + database1, convertIdValuesToDeviceID(table1, device0), "s0", Collections.singletonList("s1")) @@ -173,7 +215,7 @@ public void testLastCache() { Assert.assertFalse( cache .getLastRow( - database, + database1, convertIdValuesToDeviceID(table1, device0), "", Collections.singletonList("s1")) @@ -187,39 +229,39 @@ public void testLastCache() { updateLastCache4Query( cache, - database, + database1, convertIdValuesToDeviceID(table1, device0), new String[] {"s0", "s1", "s2"}, new TimeValuePair[] {tv0, tv1, tv2}); Assert.assertEquals( - tv0, cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s0")); + tv0, cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s0")); Assert.assertEquals( - tv1, cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s1")); + tv1, cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s1")); Assert.assertEquals( - tv2, cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s2")); + tv2, cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s2")); // Write update existing final TimeValuePair tv3 = new TimeValuePair(1L, new TsPrimitiveType.TsInt(3)); cache.updateLastCacheIfExists( - database, + database1, convertIdValuesToDeviceID(table1, device0), new String[] {"s0", "s1", "s2", "s3"}, new TimeValuePair[] {tv3, tv3, tv3, tv3}); Assert.assertEquals( - tv3, cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s0")); + tv3, cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s0")); Assert.assertEquals( - tv3, cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s1")); + tv3, cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s1")); Assert.assertEquals( - tv2, cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s2")); + tv2, cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s2")); Assert.assertNull( - cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s3")); + cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s3")); // Test null hit measurements cache.updateLastCache( - database, + database1, convertIdValuesToDeviceID(table1, device0), new String[] {"s4"}, new TimeValuePair[] {TableDeviceLastCache.EMPTY_TIME_VALUE_PAIR}); @@ -227,28 +269,28 @@ public void testLastCache() { // Miss if the "null" time value pair is not in cache, meaning that the // entry is evicted Assert.assertNull( - cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s4")); + cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s4")); // Common query updateLastCache4Query( cache, - database, + database1, convertIdValuesToDeviceID(table1, device0), new String[] {"s4"}, new TimeValuePair[] {TableDeviceLastCache.EMPTY_TIME_VALUE_PAIR}); Assert.assertSame( TableDeviceLastCache.EMPTY_TIME_VALUE_PAIR, - cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s4")); + cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s4")); // Test null miss measurements Assert.assertNull( - cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s5")); + cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s5")); // Test lastRow Optional> result = cache.getLastRow( - database, + database1, convertIdValuesToDeviceID(table1, device0), "", Collections.singletonList("s2")); @@ -256,14 +298,14 @@ public void testLastCache() { updateLastCache4Query( cache, - database, + database1, convertIdValuesToDeviceID(table1, device0), new String[] {""}, new TimeValuePair[] {new TimeValuePair(2L, TableDeviceLastCache.EMPTY_PRIMITIVE_TYPE)}); result = cache.getLastRow( - database, + database1, convertIdValuesToDeviceID(table1, device0), "", Collections.singletonList("s2")); @@ -275,7 +317,7 @@ public void testLastCache() { result = cache.getLastRow( - database, + database1, convertIdValuesToDeviceID(table1, device0), "s0", Arrays.asList("s0", "", "s1", "s4", "s5")); @@ -295,7 +337,7 @@ public void testLastCache() { // Test null source measurements result = cache.getLastRow( - database, + database1, convertIdValuesToDeviceID(table1, device0), "s4", Arrays.asList("s0", "s1", "s5")); @@ -305,17 +347,17 @@ public void testLastCache() { Assert.assertFalse( cache .getLastRow( - database, + database1, convertIdValuesToDeviceID(table1, device0), "s5", Arrays.asList("s0", "s1", "s5")) .isPresent()); final String table2 = "t2"; - cache.invalidateLastCache(database, convertIdValuesToDeviceID(table1, device0)); - cache.invalidate(database); + cache.invalidateLastCache(database1, convertIdValuesToDeviceID(table1, device0)); + cache.invalidate(database1); Assert.assertNull( - cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s2")); + cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s2")); // Invalidate table final String[] device1 = new String[] {"hebei", "p_1", "d_1"}; @@ -326,38 +368,38 @@ public void testLastCache() { updateLastCache4Query( cache, - database, + database1, convertIdValuesToDeviceID(table2, device0), tempMeasurements, tempTimeValuePairs); updateLastCache4Query( cache, - database, + database1, convertIdValuesToDeviceID(table2, device1), tempMeasurements, tempTimeValuePairs); updateLastCache4Query( cache, - database, + database1, convertIdValuesToDeviceID(table2, device2), tempMeasurements, tempTimeValuePairs); // Test cache eviction Assert.assertNull( - cache.getLastEntry(database, convertIdValuesToDeviceID(table2, device0), "s2")); + cache.getLastEntry(database1, convertIdValuesToDeviceID(table2, device0), "s2")); - cache.invalidateLastCache(database, table2); + cache.invalidateLastCache(database1, table2); Assert.assertNull( - cache.getLastEntry(database, convertIdValuesToDeviceID(table2, device1), "s2")); + cache.getLastEntry(database1, convertIdValuesToDeviceID(table2, device1), "s2")); Assert.assertNull( - cache.getLastEntry(database, convertIdValuesToDeviceID(table2, device2), "s2")); + cache.getLastEntry(database1, convertIdValuesToDeviceID(table2, device2), "s2")); // Test Long.MIN_VALUE updateLastCache4Query( cache, - database, + database1, convertIdValuesToDeviceID(table2, device0), new String[] {"", "s2"}, new TimeValuePair[] { @@ -367,7 +409,7 @@ public void testLastCache() { result = cache.getLastRow( - database, convertIdValuesToDeviceID(table2, device0), "", Arrays.asList("s2", "s3")); + database1, convertIdValuesToDeviceID(table2, device0), "", Arrays.asList("s2", "s3")); Assert.assertTrue(result.isPresent()); Assert.assertTrue(result.get().getLeft().isPresent()); Assert.assertEquals(OptionalLong.of(Long.MIN_VALUE), result.get().getLeft()); @@ -377,14 +419,14 @@ public void testLastCache() { updateLastCache4Query( cache, - database, + database1, convertIdValuesToDeviceID(table2, device0), new String[] {"s3"}, new TimeValuePair[] {new TimeValuePair(Long.MIN_VALUE, new TsPrimitiveType.TsInt(3))}); result = cache.getLastRow( - database, convertIdValuesToDeviceID(table2, device0), "s3", Arrays.asList("s2", "s3")); + database1, convertIdValuesToDeviceID(table2, device0), "s3", Arrays.asList("s2", "s3")); Assert.assertTrue(result.isPresent()); Assert.assertTrue(result.get().getLeft().isPresent()); Assert.assertEquals(OptionalLong.of(Long.MIN_VALUE), result.get().getLeft()); @@ -396,7 +438,7 @@ public void testLastCache() { result = cache.getLastRow( - database, convertIdValuesToDeviceID(table2, device0), "", Arrays.asList("s2", "s3")); + database1, convertIdValuesToDeviceID(table2, device0), "", Arrays.asList("s2", "s3")); Assert.assertTrue(result.isPresent()); Assert.assertTrue(result.get().getLeft().isPresent()); Assert.assertEquals(OptionalLong.of(Long.MIN_VALUE), result.get().getLeft()); @@ -464,35 +506,12 @@ private void updateLastCache4Query( @Test public void testIntern() { - final String database = "sg"; - final String tableName = "t"; - final List columnHeaderList = - Arrays.asList( - new ColumnHeader("hebei", TSDataType.STRING), - new ColumnHeader("p_1", TSDataType.STRING), - new ColumnHeader("d_1", TSDataType.STRING)); - final String attributeName = "attr"; - - // Prepare table - final TsTable testTable = new TsTable(tableName); - columnHeaderList.forEach( - columnHeader -> - testTable.addColumnSchema( - new IdColumnSchema(columnHeader.getColumnName(), columnHeader.getColumnType()))); - testTable.addColumnSchema(new AttributeColumnSchema(attributeName, TSDataType.STRING)); - testTable.addColumnSchema(new TimeColumnSchema("time", TSDataType.INT64)); - testTable.addColumnSchema( - new MeasurementColumnSchema( - "s1", TSDataType.BOOLEAN, TSEncoding.RLE, CompressionType.GZIP)); - DataNodeTableCache.getInstance().preUpdateTable(database, testTable); - DataNodeTableCache.getInstance().commitUpdateTable(database, tableName); - final String a = "s1"; // Different from "a" final String b = new String(a.getBytes()); Assert.assertSame( - DataNodeTableCache.getInstance().tryGetInternColumnName(database, tableName, a), - DataNodeTableCache.getInstance().tryGetInternColumnName(database, tableName, b)); + DataNodeTableCache.getInstance().tryGetInternColumnName(database1, table1, a), + DataNodeTableCache.getInstance().tryGetInternColumnName(database1, table1, b)); } } From e0fcd6073bbe947f512ab555eb7de5cfc2c70e21 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 21:57:52 +0800 Subject: [PATCH 106/109] Update TableDeviceSchemaCacheTest.java --- .../metadata/fetcher/cache/TableDeviceSchemaCacheTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java index 942fa11e8cb7..3bef1b567e6a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java @@ -110,6 +110,7 @@ public static void prepareEnvironment() { @AfterClass public static void clearEnvironment() { DataNodeTableCache.getInstance().invalid(database1); + DataNodeTableCache.getInstance().invalid(database2); config.setAllocateMemoryForSchemaCache(originMemConfig); } From 703d01c1a7adc1c357ecdad901a95bb7d66d5116 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 22:09:53 +0800 Subject: [PATCH 107/109] Update TableDeviceSchemaCacheTest.java --- .../cache/TableDeviceSchemaCacheTest.java | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java index 3bef1b567e6a..362436712f0f 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java @@ -452,11 +452,6 @@ public void testLastCache() { @Test public void testUpdateNonExistWhenWriting() { - final String database = "db1"; - final String database2 = "db2"; - - final String table1 = "t1"; - final String table2 = "t2"; final String[] device0 = new String[] {"hebei", "p_1", "d_0"}; final TimeValuePair tv3 = new TimeValuePair(1L, new TsPrimitiveType.TsInt(3)); @@ -468,7 +463,7 @@ public void testUpdateNonExistWhenWriting() { final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); cache.updateLastCacheIfExists( - database, convertIdValuesToDeviceID(table2, device0), testMeasurements, testTimeValuePairs); + database1, convertIdValuesToDeviceID(table2, device0), testMeasurements, testTimeValuePairs); cache.updateLastCacheIfExists( database2, convertIdValuesToDeviceID(table1, device0), @@ -476,23 +471,23 @@ public void testUpdateNonExistWhenWriting() { testTimeValuePairs); Assert.assertNull( - cache.getLastEntry(database, convertIdValuesToDeviceID(table2, device0), "s2")); + cache.getLastEntry(database1, convertIdValuesToDeviceID(table2, device0), "s2")); Assert.assertNull( cache.getLastEntry(database2, convertIdValuesToDeviceID(table1, device0), "s2")); updateLastCache4Query( cache, - database, + database1, convertIdValuesToDeviceID(table1, device0), new String[] {"s0"}, new TimeValuePair[] {new TimeValuePair(0L, new TsPrimitiveType.TsInt(2))}); cache.updateLastCacheIfExists( - database, convertIdValuesToDeviceID(table1, device0), testMeasurements, testTimeValuePairs); + database1, convertIdValuesToDeviceID(table1, device0), testMeasurements, testTimeValuePairs); Assert.assertEquals( - tv3, cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s0")); + tv3, cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s0")); Assert.assertNull( - cache.getLastEntry(database, convertIdValuesToDeviceID(table1, device0), "s2")); + cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s2")); } private void updateLastCache4Query( From 73ce4c337c157b36caf57148a3e4352e2096e95d Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 17 Oct 2024 22:10:15 +0800 Subject: [PATCH 108/109] Update TableDeviceSchemaCacheTest.java --- .../fetcher/cache/TableDeviceSchemaCacheTest.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java index 362436712f0f..f1807b7531d7 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCacheTest.java @@ -463,7 +463,10 @@ public void testUpdateNonExistWhenWriting() { final TableDeviceSchemaCache cache = TableDeviceSchemaCache.getInstance(); cache.updateLastCacheIfExists( - database1, convertIdValuesToDeviceID(table2, device0), testMeasurements, testTimeValuePairs); + database1, + convertIdValuesToDeviceID(table2, device0), + testMeasurements, + testTimeValuePairs); cache.updateLastCacheIfExists( database2, convertIdValuesToDeviceID(table1, device0), @@ -482,7 +485,10 @@ public void testUpdateNonExistWhenWriting() { new String[] {"s0"}, new TimeValuePair[] {new TimeValuePair(0L, new TsPrimitiveType.TsInt(2))}); cache.updateLastCacheIfExists( - database1, convertIdValuesToDeviceID(table1, device0), testMeasurements, testTimeValuePairs); + database1, + convertIdValuesToDeviceID(table1, device0), + testMeasurements, + testTimeValuePairs); Assert.assertEquals( tv3, cache.getLastEntry(database1, convertIdValuesToDeviceID(table1, device0), "s0")); From ed5230ec0e94d490693836b8444b87785d379fc2 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 18 Oct 2024 09:48:14 +0800 Subject: [PATCH 109/109] Update DropTableProcedure.java --- .../impl/schema/table/DropTableProcedure.java | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java index 78435583501d..15a3ba4ad476 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/table/DropTableProcedure.java @@ -39,6 +39,7 @@ import org.apache.iotdb.confignode.procedure.impl.schema.DataNodeRegionTaskExecutor; import org.apache.iotdb.confignode.procedure.impl.schema.SchemaUtils; import org.apache.iotdb.confignode.procedure.state.schema.DropTableState; +import org.apache.iotdb.confignode.procedure.store.ProcedureType; import org.apache.iotdb.mpp.rpc.thrift.TDeleteDataOrDevicesForDropTableReq; import org.apache.iotdb.mpp.rpc.thrift.TInvalidateTableCacheReq; import org.apache.iotdb.rpc.TSStatusCode; @@ -260,20 +261,16 @@ protected DropTableState getInitialState() { return DropTableState.CHECK_AND_INVALIDATE_TABLE; } + @Override + public void serialize(final DataOutputStream stream) throws IOException { + stream.writeShort(ProcedureType.DROP_TABLE_PROCEDURE.getTypeCode()); + super.serialize(stream); + } + private class DropTableRegionTaskExecutor extends DataNodeRegionTaskExecutor { private final String taskName; - DropTableRegionTaskExecutor( - final String taskName, - final ConfigNodeProcedureEnv env, - final Map targetSchemaRegionGroup, - final CnToDnAsyncRequestType dataNodeRequestType, - final BiFunction, Q> dataNodeRequestGenerator) { - super(env, targetSchemaRegionGroup, false, dataNodeRequestType, dataNodeRequestGenerator); - this.taskName = taskName; - } - DropTableRegionTaskExecutor( final String taskName, final ConfigNodeProcedureEnv env,