From 514e79c1d4e0abc2e418624dc63c50506de8de83 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 12 Nov 2024 12:01:14 +0800 Subject: [PATCH 01/31] [fix](ngram bloomfilter) fix narrow conversion for ngram bf_size #43480 (#43655) cherry pick from #43480 --- .../olap/rowset/segment_v2/segment_writer.cpp | 15 +++++- .../org/apache/doris/analysis/IndexDef.java | 4 +- .../test_ngram_bloomfilter_index.groovy | 47 +++++++++++++++++++ 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index 6aab6c464fd784..01e818b454d14a 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -164,8 +164,19 @@ Status SegmentWriter::init(const std::vector& col_ids, bool has_key, if (tablet_index) { opts.need_bloom_filter = true; opts.is_ngram_bf_index = true; - opts.gram_size = tablet_index->get_gram_size(); - opts.gram_bf_size = tablet_index->get_gram_bf_size(); + //narrow convert from int32_t to uint8_t and uint16_t which is dangerous + auto gram_size = tablet_index->get_gram_size(); + auto gram_bf_size = tablet_index->get_gram_bf_size(); + if (gram_size > 256 || gram_size < 1) { + return Status::NotSupported("Do not support ngram bloom filter for ngram_size: ", + gram_size); + } + if (gram_bf_size > 65535 || gram_bf_size < 64) { + return Status::NotSupported("Do not support ngram bloom filter for bf_size: ", + gram_bf_size); + } + opts.gram_size = gram_size; + opts.gram_bf_size = gram_bf_size; } opts.need_bitmap_index = column.has_bitmap_index(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java index 19d7681ffed2c9..406ee0cb247660 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IndexDef.java @@ -252,8 +252,8 @@ public void checkColumn(Column column, KeysType keysType, boolean enableUniqueKe if (ngramSize > 256 || ngramSize < 1) { throw new AnalysisException("gram_size should be integer and less than 256"); } - if (bfSize > 65536 || bfSize < 64) { - throw new AnalysisException("bf_size should be integer and between 64 and 65536"); + if (bfSize > 65535 || bfSize < 64) { + throw new AnalysisException("bf_size should be integer and between 64 and 65535"); } } catch (NumberFormatException e) { throw new AnalysisException("invalid ngram properties:" + e.getMessage(), e); diff --git a/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy b/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy index c56eed967a08b0..e2ab9b9c117f1c 100644 --- a/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy +++ b/regression-test/suites/index_p0/test_ngram_bloomfilter_index.groovy @@ -59,4 +59,51 @@ suite("test_ngram_bloomfilter_index") { qt_select_eq_3 "SELECT * FROM ${tableName} WHERE http_url = '/%/7212503657802320699%' ORDER BY key_id" qt_select_in_3 "SELECT * FROM ${tableName} WHERE http_url IN ('/%/7212503657802320699%') ORDER BY key_id" qt_select_like_3 "SELECT * FROM ${tableName} WHERE http_url like '/%/7212503657802320699%' ORDER BY key_id" + + //case for bf_size 65536 + def tableName2 = 'test_ngram_bloomfilter_index2' + sql "DROP TABLE IF EXISTS ${tableName2}" + test { + sql """ + CREATE TABLE IF NOT EXISTS ${tableName2} ( + `key_id` bigint(20) NULL COMMENT '', + `category` varchar(200) NULL COMMENT '', + `https_url` varchar(300) NULL COMMENT '', + `hostname` varchar(300) NULL, + `http_url` text NULL COMMENT '', + `url_path` varchar(2000) NULL COMMENT '', + `cnt` bigint(20) NULL COMMENT '', + `host_flag` boolean NULL COMMENT '', + INDEX idx_ngrambf (`http_url`) USING NGRAM_BF PROPERTIES("gram_size" = "2", "bf_size" = "65536") + ) ENGINE=OLAP + DUPLICATE KEY(`key_id`, `category`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`key_id`) BUCKETS 3 + PROPERTIES("replication_num" = "1"); + """ + exception "bf_size should be integer and between 64 and 65535" + } + + def tableName3 = 'test_ngram_bloomfilter_index3' + sql "DROP TABLE IF EXISTS ${tableName3}" + sql """ + CREATE TABLE IF NOT EXISTS ${tableName3} ( + `key_id` bigint(20) NULL COMMENT '', + `category` varchar(200) NULL COMMENT '', + `https_url` varchar(300) NULL COMMENT '', + `hostname` varchar(300) NULL, + `http_url` text NULL COMMENT '', + `url_path` varchar(2000) NULL COMMENT '', + `cnt` bigint(20) NULL COMMENT '', + `host_flag` boolean NULL COMMENT '' + ) ENGINE=OLAP + DUPLICATE KEY(`key_id`, `category`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`key_id`) BUCKETS 3 + PROPERTIES("replication_num" = "1"); + """ + test { + sql """ALTER TABLE ${tableName3} ADD INDEX idx_http_url(http_url) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="65536") COMMENT 'http_url ngram_bf index'""" + exception "bf_size should be integer and between 64 and 65535" + } } From c67fb4eb48e7d72ec13e92a981a53bbc423227c4 Mon Sep 17 00:00:00 2001 From: walter Date: Tue, 12 Nov 2024 14:03:04 +0800 Subject: [PATCH 02/31] [fix](case) Fix test_backup_restore_atomic_with_alter with SYNC #43601 (#43647) cherry pick from #43601 --- .../test_backup_restore_atomic_with_alter.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/regression-test/suites/backup_restore/test_backup_restore_atomic_with_alter.groovy b/regression-test/suites/backup_restore/test_backup_restore_atomic_with_alter.groovy index b66ff391f1501f..8974b87c09a82b 100644 --- a/regression-test/suites/backup_restore/test_backup_restore_atomic_with_alter.groovy +++ b/regression-test/suites/backup_restore/test_backup_restore_atomic_with_alter.groovy @@ -108,6 +108,7 @@ suite("test_backup_restore_atomic_with_alter", "backup_restore") { "atomic_restore" = "true" ) """ + sql "SYNC" boolean restore_paused = false for (int k = 0; k < 60; k++) { @@ -121,6 +122,8 @@ suite("test_backup_restore_atomic_with_alter", "backup_restore") { } assertTrue(restore_paused) + sql "SYNC" + // 0. table_1 has in_atomic_restore property def show_result = sql """ SHOW CREATE TABLE ${dbName}.${tableNamePrefix}_1 """ logger.info("SHOW CREATE TABLE ${tableNamePrefix}_1: ${show_result}") @@ -224,6 +227,7 @@ suite("test_backup_restore_atomic_with_alter", "backup_restore") { sql "CANCEL RESTORE FROM ${dbName}" + sql "SYNC" // 5. The restore job is cancelled, the in_atomic_restore property has been removed. show_result = sql """ SHOW CREATE TABLE ${dbName}.${tableNamePrefix}_1 """ From d62a765190f296f5b69ade870f2874b81b9a1304 Mon Sep 17 00:00:00 2001 From: walter Date: Wed, 13 Nov 2024 12:33:51 +0800 Subject: [PATCH 03/31] [opt](binlog) Support modify comment binlog #39783 (#43825) cherry pick from #39783 Co-authored-by: smallx --- .../java/org/apache/doris/binlog/BinlogManager.java | 10 ++++++++++ .../main/java/org/apache/doris/persist/EditLog.java | 4 +++- .../doris/persist/ModifyCommentOperationLog.java | 4 ++++ gensrc/thrift/FrontendService.thrift | 1 + 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java index 47b0bb3c7670f5..4f67f663879875 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java @@ -29,6 +29,7 @@ import org.apache.doris.persist.BatchModifyPartitionsInfo; import org.apache.doris.persist.BinlogGcInfo; import org.apache.doris.persist.DropPartitionInfo; +import org.apache.doris.persist.ModifyCommentOperationLog; import org.apache.doris.persist.ModifyTablePropertyOperationLog; import org.apache.doris.persist.ReplacePartitionOperationLog; import org.apache.doris.persist.TableAddOrDropColumnsInfo; @@ -375,6 +376,15 @@ public void addColumnRename(TableRenameColumnInfo info, long commitSeq) { addBarrierLog(log, commitSeq); } + public void addModifyComment(ModifyCommentOperationLog info, long commitSeq) { + long dbId = info.getDbId(); + long tableId = info.getTblId(); + TBinlogType type = TBinlogType.MODIFY_COMMENT; + String data = info.toJson(); + BarrierLog log = new BarrierLog(dbId, tableId, type, data); + addBarrierLog(log, commitSeq); + } + // get the dropped partitions of the db. public List getDroppedPartitions(long dbId) { lock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 963a1e06c8a6f6..7fdac2ebfaa8c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -1739,7 +1739,9 @@ public void logBatchRemoveTransactions(BatchRemoveTransactionsOperationV2 op) { } public void logModifyComment(ModifyCommentOperationLog op) { - logEdit(OperationType.OP_MODIFY_COMMENT, op); + long logId = logEdit(OperationType.OP_MODIFY_COMMENT, op); + LOG.info("log modify comment, logId : {}, infos: {}", logId, op); + Env.getCurrentEnv().getBinlogManager().addModifyComment(op, logId); } public void logCreateSqlBlockRule(SqlBlockRule rule) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyCommentOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyCommentOperationLog.java index 0b6f1f984507e0..f9bb7f5084e750 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyCommentOperationLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/ModifyCommentOperationLog.java @@ -94,4 +94,8 @@ public static ModifyCommentOperationLog read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, ModifyCommentOperationLog.class); } + + public String toJson() { + return GsonUtils.GSON.toJson(this); + } } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 575643030e11ca..2739b1c209607c 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1029,6 +1029,7 @@ enum TBinlogType { TRUNCATE_TABLE = 13, RENAME_TABLE = 14, RENAME_COLUMN = 15, + MODIFY_COMMENT = 16, } struct TBinlog { From 294e7fee9b2bae6b62f114b78f84769cd441fdf8 Mon Sep 17 00:00:00 2001 From: starocean999 Date: Wed, 13 Nov 2024 17:49:28 +0800 Subject: [PATCH 04/31] [fix](planner) NullLiteral should always having a correct Type and set to be analyzed (#43372) pick from master https://github.com/apache/doris/pull/43370 --- .../apache/doris/analysis/NullLiteral.java | 1 + .../apache/doris/analysis/StringLiteral.java | 2 +- .../data/insert_p0/test_insert_nan.out | 4 +++ .../suites/insert_p0/test_insert_nan.groovy | 34 +++++++++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 regression-test/data/insert_p0/test_insert_nan.out create mode 100644 regression-test/suites/insert_p0/test_insert_nan.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java index 7d0e38c5b6045a..d4c52dd71aadbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NullLiteral.java @@ -52,6 +52,7 @@ public NullLiteral() { public static NullLiteral create(Type type) { NullLiteral l = new NullLiteral(); l.type = type; + l.analysisDone(); return l; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index 542c668459af51..a81e569f3ce66f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -243,7 +243,7 @@ protected Expr uncheckedCastTo(Type targetType) throws AnalysisException { return new FloatLiteral(Double.valueOf(value), targetType); } catch (NumberFormatException e) { // consistent with CastExpr's getResultValue() method - return new NullLiteral(); + return NullLiteral.create(targetType); } case DECIMALV2: case DECIMAL32: diff --git a/regression-test/data/insert_p0/test_insert_nan.out b/regression-test/data/insert_p0/test_insert_nan.out new file mode 100644 index 00000000000000..f0d5ba9c2d6c85 --- /dev/null +++ b/regression-test/data/insert_p0/test_insert_nan.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +1 \N + diff --git a/regression-test/suites/insert_p0/test_insert_nan.groovy b/regression-test/suites/insert_p0/test_insert_nan.groovy new file mode 100644 index 00000000000000..8650c5d0cf1ef3 --- /dev/null +++ b/regression-test/suites/insert_p0/test_insert_nan.groovy @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_insert_nan") { + sql """drop table if exists `nan_table`;""" + + sql """ + CREATE TABLE `nan_table` ( + `id` int NOT NULL , + `val` double, + ) ENGINE=OLAP + UNIQUE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 3 + PROPERTIES("replication_num" = "1"); + """ + + sql """insert into nan_table values ('1', 'nan');""" + + qt_select """select * from nan_table;""" +} From 9c74a96d42d5e23fc4e877695d384efe1f75d2ac Mon Sep 17 00:00:00 2001 From: walter Date: Wed, 13 Nov 2024 18:31:42 +0800 Subject: [PATCH 05/31] [feat](backup) GetSnapshot returns snapshot expiration time #43731 (#43863) cherry pick from #43731 --- .../java/org/apache/doris/backup/BackupJob.java | 8 +++++++- .../java/org/apache/doris/backup/Snapshot.java | 17 ++++++++++++++--- .../doris/service/FrontendServiceImpl.java | 9 +++++++-- gensrc/thrift/FrontendService.thrift | 1 + gensrc/thrift/Status.thrift | 7 +++++++ .../apache/doris/regression/suite/Syncer.groovy | 2 ++ 6 files changed, 38 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index dc92e9a07c3c1f..8a5043ebe7f61a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -978,12 +978,18 @@ public synchronized Snapshot getSnapshot() { return null; } + // Avoid loading expired meta. + long expiredAt = createTime + timeoutMs; + if (System.currentTimeMillis() >= expiredAt) { + return new Snapshot(label, new byte[0], new byte[0], expiredAt); + } + try { File metaInfoFile = new File(localMetaInfoFilePath); File jobInfoFile = new File(localJobInfoFilePath); byte[] metaInfoBytes = Files.readAllBytes(metaInfoFile.toPath()); byte[] jobInfoBytes = Files.readAllBytes(jobInfoFile.toPath()); - return new Snapshot(label, metaInfoBytes, jobInfoBytes); + return new Snapshot(label, metaInfoBytes, jobInfoBytes, expiredAt); } catch (IOException e) { LOG.warn("failed to load meta info and job info file, meta info file {}, job info file {}: ", localMetaInfoFilePath, localJobInfoFilePath, e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java index e31309b19a59ee..c4c93548177ca5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java @@ -31,16 +31,19 @@ public class Snapshot { @SerializedName(value = "jobInfo") private byte[] jobInfo = null; + @SerializedName(value = "expired_at") + private long expiredAt = 0; + public Snapshot() { } - public Snapshot(String label, byte[] meta, byte[] jobInfo) { + public Snapshot(String label, byte[] meta, byte[] jobInfo, long expiredAt) { this.label = label; this.meta = meta; this.jobInfo = jobInfo; + this.expiredAt = expiredAt; } - public byte[] getMeta() { return meta; } @@ -49,17 +52,25 @@ public byte[] getJobInfo() { return jobInfo; } + public long getExpiredAt() { + return expiredAt; + } + + public boolean isExpired() { + return System.currentTimeMillis() > expiredAt; + } + public String toJson() { return GsonUtils.GSON.toJson(this); } @Override public String toString() { - // return toJson(); return "Snapshot{" + "label='" + label + '\'' + ", meta=" + meta + ", jobInfo=" + jobInfo + + ", expiredAt=" + expiredAt + '}'; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index d83ff7e08156d1..8c521cc73b0b52 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -2813,12 +2813,16 @@ private TGetSnapshotResult getSnapshotImpl(TGetSnapshotRequest request, String c if (snapshot == null) { result.getStatus().setStatusCode(TStatusCode.SNAPSHOT_NOT_EXIST); result.getStatus().addToErrorMsgs(String.format("snapshot %s not exist", label)); + } else if (snapshot.isExpired()) { + result.getStatus().setStatusCode(TStatusCode.SNAPSHOT_EXPIRED); + result.getStatus().addToErrorMsgs(String.format("snapshot %s is expired", label)); } else { byte[] meta = snapshot.getMeta(); byte[] jobInfo = snapshot.getJobInfo(); + long expiredAt = snapshot.getExpiredAt(); - LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info size: {}", - label, meta.length, jobInfo.length); + LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info size: {}, expired at: {}", + label, meta.length, jobInfo.length, expiredAt); if (request.isEnableCompress()) { meta = GZIPUtils.compress(meta); jobInfo = GZIPUtils.compress(jobInfo); @@ -2830,6 +2834,7 @@ private TGetSnapshotResult getSnapshotImpl(TGetSnapshotRequest request, String c } result.setMeta(meta); result.setJobInfo(jobInfo); + result.setExpiredAt(expiredAt); } return result; diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 2739b1c209607c..6f9242cdd0efa4 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1087,6 +1087,7 @@ struct TGetSnapshotResult { 3: optional binary job_info 4: optional Types.TNetworkAddress master_address 5: optional bool compressed; + 6: optional i64 expiredAt; // in millis } struct TTableRef { diff --git a/gensrc/thrift/Status.thrift b/gensrc/thrift/Status.thrift index 2cc9adb344a3f8..bb3450dd120c4e 100644 --- a/gensrc/thrift/Status.thrift +++ b/gensrc/thrift/Status.thrift @@ -103,6 +103,13 @@ enum TStatusCode { TABLET_MISSING = 72, NOT_MASTER = 73, + + OBTAIN_LOCK_FAILED = 74, + + SNAPSHOT_EXPIRED = 75, + + // Not be larger than 200, see status.h + // And all error code defined here, should also be defined in status.h } struct TStatus { diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy index 20a4cf1ffd3815..398f1ec262485f 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy @@ -459,6 +459,8 @@ class Syncer { logger.error("TGetSnapshotResult meta is unset.") } else if (!result.isSetJobInfo()) { logger.error("TGetSnapshotResult job info is unset.") + } else if (!result.isSetExpiredAt()) { + logger.error("TGetSnapshotResult expiredAt is unset.") } else { isCheckedOK = true } From a6a46f2b300c9892c5ee1b006735e8b00ad050ba Mon Sep 17 00:00:00 2001 From: starocean999 Date: Thu, 14 Nov 2024 09:53:47 +0800 Subject: [PATCH 06/31] [fix](planner)show tablet command return wrong result when having limit and offset (#43905) pick from master https://github.com/apache/doris/pull/43768 --- .../org/apache/doris/qe/ShowExecutor.java | 42 ++++++++++--------- .../suites/show_p0/test_show_tablet.groovy | 36 ++++++++++++++++ 2 files changed, 58 insertions(+), 20 deletions(-) create mode 100644 regression-test/suites/show_p0/test_show_tablet.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java index 69c92d0f1668e6..7156d3508823bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java @@ -1890,30 +1890,32 @@ private void handleShowTablet() throws AnalysisException { } } } - if (sizeLimit > -1 && tabletInfos.size() < sizeLimit) { + if (showStmt.hasOffset() && showStmt.getOffset() >= tabletInfos.size()) { tabletInfos.clear(); - } else if (sizeLimit > -1) { - tabletInfos = tabletInfos.subList((int) showStmt.getOffset(), (int) sizeLimit); - } - - // order by - List orderByPairs = showStmt.getOrderByPairs(); - ListComparator> comparator = null; - if (orderByPairs != null) { - OrderByPair[] orderByPairArr = new OrderByPair[orderByPairs.size()]; - comparator = new ListComparator<>(orderByPairs.toArray(orderByPairArr)); } else { - // order by tabletId, replicaId - comparator = new ListComparator<>(0, 1); - } - Collections.sort(tabletInfos, comparator); + // order by + List orderByPairs = showStmt.getOrderByPairs(); + ListComparator> comparator = null; + if (orderByPairs != null) { + OrderByPair[] orderByPairArr = new OrderByPair[orderByPairs.size()]; + comparator = new ListComparator<>(orderByPairs.toArray(orderByPairArr)); + } else { + // order by tabletId, replicaId + comparator = new ListComparator<>(0, 1); + } + Collections.sort(tabletInfos, comparator); + if (sizeLimit > -1) { + tabletInfos = tabletInfos.subList((int) showStmt.getOffset(), + Math.min((int) sizeLimit, tabletInfos.size())); + } - for (List tabletInfo : tabletInfos) { - List oneTablet = new ArrayList(tabletInfo.size()); - for (Comparable column : tabletInfo) { - oneTablet.add(column.toString()); + for (List tabletInfo : tabletInfos) { + List oneTablet = new ArrayList(tabletInfo.size()); + for (Comparable column : tabletInfo) { + oneTablet.add(column.toString()); + } + rows.add(oneTablet); } - rows.add(oneTablet); } } finally { olapTable.readUnlock(); diff --git a/regression-test/suites/show_p0/test_show_tablet.groovy b/regression-test/suites/show_p0/test_show_tablet.groovy new file mode 100644 index 00000000000000..fca748b876cb4d --- /dev/null +++ b/regression-test/suites/show_p0/test_show_tablet.groovy @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_show_tablet") { + sql """drop table if exists show_tablets_test_t;""" + sql """create table show_tablets_test_t ( + id BIGINT, + username VARCHAR(20) + ) + DISTRIBUTED BY HASH(id) BUCKETS 5 + PROPERTIES ( + "replication_num" = "1" + );""" + def res = sql """SHOW TABLETS FROM show_tablets_test_t limit 5, 1;""" + assertTrue(res.size() == 0) + + res = sql """SHOW TABLETS FROM show_tablets_test_t limit 3, 5;""" + assertTrue(res.size() == 2) + + res = sql """SHOW TABLETS FROM show_tablets_test_t limit 10;""" + assertTrue(res.size() == 5) +} From 466c309305382d079985ab1e3ac4587d1b164380 Mon Sep 17 00:00:00 2001 From: minghong Date: Fri, 15 Nov 2024 10:28:57 +0800 Subject: [PATCH 07/31] [fix](nereids) convert stringLikeLiteral to double should use byte length (#43776) branch-2.0 (#43910) ### What problem does this PR solve? pick #43776 The optimizer maps different data types to Double, allowing for a unified comparison of literals. StringLikeLiteral is regarded as a long int, and map the long int to double. To extract the first N bytes from a string, you should use String.getBytes().length instead of String.length(). --- .../literal/StringLikeLiteral.java | 5 +-- .../literal/StringLikeLiteralTest.java | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java index 30cb95801954ea..a1c0fde10e71fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java @@ -46,11 +46,12 @@ public double getDouble() { * get double value */ public static double getDouble(String str) { + byte[] bytes = str.getBytes(); long v = 0; int pos = 0; - int len = Math.min(str.length(), 7); + int len = Math.min(bytes.length, 7); while (pos < len) { - v += Byte.toUnsignedLong(str.getBytes()[pos]) << ((6 - pos) * 8); + v += Byte.toUnsignedLong(bytes[pos]) << ((6 - pos) * 8); pos++; } return (double) v; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java new file mode 100644 index 00000000000000..c1e9bc0e839b2a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteralTest.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.literal; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class StringLikeLiteralTest { + @Test + public void testStrToDouble() { + // fix bug: maxStr.length = 4, maxStr.getBytes().length=12 + // when converting str to double, bytes length is used instead of string length + String minStr = "商家+店长+场地+设备类型维度"; + String maxStr = "商家维度"; + double d1 = StringLikeLiteral.getDouble(minStr); + double d2 = StringLikeLiteral.getDouble(maxStr); + Assertions.assertTrue(d1 < d2); + } +} From c2732d4f30c6388edbee3c830a1ce8388f0cd698 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Sat, 16 Nov 2024 10:06:34 +0800 Subject: [PATCH 08/31] [fix](memory) Fix jemalloc hook deadlock at BE start (#44009) It should be caused by the third-party library, but in any case, this fix should be possible. ``` #0 0x00007f738432318c in __lll_lock_wait_private () from /lib64/libc.so.6 #1 0x00007f73843505ed in __internal_atexit () from /lib64/libc.so.6 #2 0x0000563f6a54db30 in malloc () #3 0x0000563f74e82b08 in operator new(unsigned long) () #4 0x0000563f6a6144bd in doris::ThreadContextPtr::ThreadContextPtr() () #5 0x0000563f6a54e21c in doris_calloc () #6 0x00007f7384350557 in __new_exitfn () from /lib64/libc.so.6 #7 0x00007f73843505fc in __internal_atexit () from /lib64/libc.so.6 #8 0x0000563f72c79c03 in ?? () #9 0x0000563f74e3b5fd in __libc_csu_init () #10 0x00007f7384339d18 in __libc_start_main () from /lib64/libc.so.6 #11 0x0000563f69ae302a in _start () ``` --- be/src/common/daemon.cpp | 1 + be/src/common/daemon.h | 1 + be/src/runtime/thread_context.h | 9 +++++---- be/src/service/doris_main.cpp | 2 ++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index f91aec88f59572..98315609865de0 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -409,6 +409,7 @@ static void init_doris_metrics(const std::vector& store_paths) { void signal_handler(int signal) { if (signal == SIGINT || signal == SIGTERM) { k_doris_exit = true; + k_doris_start = false; LOG(INFO) << "doris start to exit"; } } diff --git a/be/src/common/daemon.h b/be/src/common/daemon.h index 5ac9abbe2b15ed..36d2ca0813a62e 100644 --- a/be/src/common/daemon.h +++ b/be/src/common/daemon.h @@ -27,6 +27,7 @@ namespace doris { struct StorePath; inline bool k_doris_exit = false; +inline bool k_doris_start = false; class Daemon { public: diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h index bbfd1a029797e5..23ee742b867d66 100644 --- a/be/src/runtime/thread_context.h +++ b/be/src/runtime/thread_context.h @@ -100,6 +100,7 @@ class MemTracker; class RuntimeState; extern bool k_doris_exit; +extern bool k_doris_start; extern bthread_key_t btls_key; // Using gcc11 compiles thread_local variable on lower versions of GLIBC will report an error, @@ -388,17 +389,17 @@ class AddThreadMemTrackerConsumer { // which is different from the previous behavior. #define CONSUME_MEM_TRACKER(size) \ do { \ - if (doris::thread_context_ptr.init) { \ + if (doris::k_doris_start && doris::thread_context_ptr.init) { \ doris::thread_context()->consume_memory(size); \ - } else if (doris::ExecEnv::GetInstance()->initialized()) { \ + } else if (doris::k_doris_start && doris::ExecEnv::GetInstance()->initialized()) { \ doris::ExecEnv::GetInstance()->orphan_mem_tracker_raw()->consume_no_update_peak(size); \ } \ } while (0) #define RELEASE_MEM_TRACKER(size) \ do { \ - if (doris::thread_context_ptr.init) { \ + if (doris::k_doris_start && doris::thread_context_ptr.init) { \ doris::thread_context()->consume_memory(-size); \ - } else if (doris::ExecEnv::GetInstance()->initialized()) { \ + } else if (doris::k_doris_start && doris::ExecEnv::GetInstance()->initialized()) { \ doris::ExecEnv::GetInstance()->orphan_mem_tracker_raw()->consume_no_update_peak( \ -size); \ } \ diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index 3b6cf2969cfec6..643ba0837b9591 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -82,6 +82,7 @@ int __llvm_profile_write_file(); namespace doris { extern bool k_doris_exit; +extern bool k_doris_start; static void thrift_output(const char* x) { LOG(WARNING) << "thrift internal message: " << x; @@ -463,6 +464,7 @@ int main(int argc, char** argv) { // init exec env auto exec_env = doris::ExecEnv::GetInstance(); + doris::k_doris_start = true; doris::ExecEnv::init(exec_env, paths); doris::TabletSchemaCache::create_global_schema_cache(); From f20aff5dffc9b741884fda14b77fb6d5699a77f3 Mon Sep 17 00:00:00 2001 From: James Date: Sat, 16 Nov 2024 23:21:43 +0800 Subject: [PATCH 09/31] [improvement](statistics)Skip auto analyze empty table. (#43865) (#44045) backport: https://github.com/apache/doris/pull/43865 --- .../doris/statistics/AnalysisManager.java | 3 ++ .../statistics/StatisticsAutoCollector.java | 15 +++++++-- .../doris/statistics/TableStatsMeta.java | 4 +++ .../StatisticsAutoCollectorTest.java | 15 +++++++++ .../suites/statistics/test_analyze_mv.groovy | 33 +++++++++++++++++++ 5 files changed, 68 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java index af17a63da20816..b78914442d3e83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java @@ -219,6 +219,9 @@ public void createAnalysisJob(AnalyzeTblStmt stmt, boolean proxy) throws DdlExce } List jobs = new ArrayList<>(); autoCollector.createAnalyzeJobForTbl(stmt.getDb(), jobs, stmt.getTable()); + if (jobs.isEmpty()) { + return; + } AnalysisInfo job = autoCollector.getReAnalyzeRequiredPart(jobs.get(0)); if (job != null) { Env.getCurrentEnv().getStatisticsAutoCollector().createSystemAnalysisJob(job); diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java index b4d8f1ad0eca64..cf9a52da0d75d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java @@ -144,7 +144,6 @@ protected List constructAnalysisInfo(DatabaseIf } catch (Throwable t) { LOG.warn("Failed to analyze table {}.{}.{}", db.getCatalog().getName(), db.getFullName(), table.getName(), t); - continue; } } return analysisInfos; @@ -186,7 +185,19 @@ protected void createAnalyzeJobForTbl(DatabaseIf db, return; } } - long rowCount = StatisticsUtil.isEmptyTable(table, analysisMethod) ? 0 : table.getRowCount(); + // We don't auto analyze empty table to avoid all 0 stats. + // Because all 0 is more dangerous than unknown stats when row count report is delayed. + AnalysisManager manager = Env.getServingEnv().getAnalysisManager(); + TableStatsMeta tableStatsStatus = manager.findTableStatsStatus(table.getId()); + long rowCount = table.getRowCount(); + if (rowCount <= 0) { + LOG.info("Table {} is empty, remove its old stats and skip auto analyze it.", table.getName()); + // Remove the table's old stats if exists. + if (tableStatsStatus != null && !tableStatsStatus.isColumnsStatsEmpty()) { + manager.dropStats(table); + } + return; + } AnalysisInfo jobInfo = new AnalysisInfoBuilder() .setJobId(Env.getCurrentEnv().getNextId()) .setCatalogId(db.getCatalog().getId()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java index f39ad452e0364c..f24ffba5234b40 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java @@ -242,4 +242,8 @@ protected void clearStaleIndexRowCount(OlapTable table) { protected void addIndexRowForTest(long indexId, long rowCount) { indexesRowCount.put(indexId, rowCount); } + + public boolean isColumnsStatsEmpty() { + return colNameToColStatsMeta == null || colNameToColStatsMeta.isEmpty(); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java index 9f2003535e1f4a..ba50a494947940 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java @@ -142,6 +142,11 @@ public List getSchemaAllIndexes(boolean full) { columns.add(new Column("c2", PrimitiveType.HLL)); return columns; } + + @Mock + public long getRowCount() { + return 1; + } }; StatisticsAutoCollector saa = new StatisticsAutoCollector(); List analysisInfoList = saa.constructAnalysisInfo(new Database(1, "anydb")); @@ -397,6 +402,11 @@ public List getMvColumnIndexIds(String columnName) { objects.add(-1L); return objects; } + + @Mock + public long getRowCount() { + return 1; + } }; new MockUp() { @@ -469,6 +479,11 @@ public List getMvColumnIndexIds(String columnName) { objects.add(-1L); return objects; } + + @Mock + public long getRowCount() { + return 1; + } }; new MockUp() { diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy b/regression-test/suites/statistics/test_analyze_mv.groovy index e3045f0301896d..5ae7a5ec83fdb1 100644 --- a/regression-test/suites/statistics/test_analyze_mv.groovy +++ b/regression-test/suites/statistics/test_analyze_mv.groovy @@ -689,6 +689,18 @@ suite("test_analyze_mv") { assertEquals("0", result_row[0][3]) assertEquals("-1", result_row[0][4]) + // ** Embedded test for skip auto analyze when table is empty + sql """analyze table mvTestDup properties ("use.auto.analyzer" = "true")""" + def empty_test = sql """show auto analyze mvTestDup""" + assertEquals(0, empty_test.size()) + empty_test = sql """show column stats mvTestDup""" + assertEquals(0, empty_test.size()) + // ** End of embedded test + + sql """analyze table mvTestDup with sync""" + empty_test = sql """show column stats mvTestDup""" + assertEquals(12, empty_test.size()) + for (int i = 0; i < 120; i++) { result_row = sql """show index stats mvTestDup mv3""" logger.info("mv3 stats: " + result_row) @@ -703,6 +715,27 @@ suite("test_analyze_mv") { assertEquals("mv3", result_row[0][1]) assertEquals("0", result_row[0][3]) assertEquals("0", result_row[0][4]) + + // ** Embedded test for skip auto analyze when table is empty again + sql """analyze table mvTestDup properties ("use.auto.analyzer" = "true")""" + empty_test = sql """show auto analyze mvTestDup""" + assertEquals(0, empty_test.size()) + empty_test = sql """show column stats mvTestDup""" + for (int i = 0; i < 100; i++) { + empty_test = sql """show column stats mvTestDup""" + if (empty_test.size() != 0) { + logger.info("async delete is not finished yet.") + Thread.sleep(1000) + } + break + } + assertEquals(0, empty_test.size()) + // ** End of embedded test + + sql """analyze table mvTestDup with sync""" + empty_test = sql """show column stats mvTestDup""" + assertEquals(12, empty_test.size()) + sql """insert into mvTestDup values (1, 2, 3, 4, 5), (1, 2, 3, 4, 5), (10, 20, 30, 40, 50), (10, 20, 30, 40, 50), (100, 200, 300, 400, 500), (1001, 2001, 3001, 4001, 5001);""" result_row = sql """show index stats mvTestDup mv3""" assertEquals(1, result_row.size()) From 514a17e5d7742c3a35f02d8ff0c3d64d6958960d Mon Sep 17 00:00:00 2001 From: walter Date: Mon, 18 Nov 2024 15:05:57 +0800 Subject: [PATCH 10/31] [improve](http) Save the requested url in http execution error #43855 (#44105) cherry pick from #43855 --- be/src/http/http_client.cpp | 22 +++++++++++--- be/src/http/http_client.h | 3 +- be/src/olap/single_replica_compaction.cpp | 12 ++++---- be/src/olap/single_replica_compaction.h | 4 +-- be/src/olap/task/engine_clone_task.cpp | 17 ++++------- be/src/olap/task/engine_clone_task.h | 2 -- be/src/util/security.h | 35 +++++++++++++++++++++++ 7 files changed, 70 insertions(+), 25 deletions(-) create mode 100644 be/src/util/security.h diff --git a/be/src/http/http_client.cpp b/be/src/http/http_client.cpp index 3cad98e11b700f..3fc2fc85a251c8 100644 --- a/be/src/http/http_client.cpp +++ b/be/src/http/http_client.cpp @@ -26,6 +26,7 @@ #include "common/config.h" #include "http/http_headers.h" #include "http/http_status.h" +#include "util/security.h" #include "util/stack_util.h" namespace doris { @@ -198,9 +199,11 @@ Status HttpClient::execute(const std::function& callback) { Status status; @@ -286,7 +298,9 @@ Status HttpClient::execute_with_retry(int retry_times, int sleep_time, if (http_status == 200) { return status; } else { - auto error_msg = fmt::format("http status code is not 200, code={}", http_status); + std::string url = mask_token(client._get_url()); + auto error_msg = fmt::format("http status code is not 200, code={}, url={}", + http_status, url); LOG(WARNING) << error_msg; return Status::HttpError(error_msg); } diff --git a/be/src/http/http_client.h b/be/src/http/http_client.h index a6ab49c1e8af21..a5fe2d3f48a283 100644 --- a/be/src/http/http_client.h +++ b/be/src/http/http_client.h @@ -147,7 +147,8 @@ class HttpClient { size_t on_response_data(const void* data, size_t length); private: - const char* _to_errmsg(CURLcode code); + const char* _to_errmsg(CURLcode code) const; + const char* _get_url() const; private: CURL* _curl = nullptr; diff --git a/be/src/olap/single_replica_compaction.cpp b/be/src/olap/single_replica_compaction.cpp index fdccc78816f09b..c190796f6bc3b3 100644 --- a/be/src/olap/single_replica_compaction.cpp +++ b/be/src/olap/single_replica_compaction.cpp @@ -37,6 +37,7 @@ #include "task/engine_clone_task.h" #include "util/brpc_client_cache.h" #include "util/doris_metrics.h" +#include "util/security.h" #include "util/thrift_rpc_helper.h" #include "util/trace.h" @@ -390,7 +391,7 @@ Status SingleReplicaCompaction::_download_files(DataDir* data_dir, // then it will try to clone from BE 2, but it will find the file 1 already exist, but file 1 with same // name may have different versions. VLOG_DEBUG << "single replica compaction begin to download files, remote path=" - << remote_url_prefix << " local_path=" << local_path; + << mask_token(remote_url_prefix) << " local_path=" << local_path; RETURN_IF_ERROR(io::global_local_filesystem()->delete_directory(local_path)); RETURN_IF_ERROR(io::global_local_filesystem()->create_directory(local_path)); @@ -448,9 +449,9 @@ Status SingleReplicaCompaction::_download_files(DataDir* data_dir, std::string local_file_path = local_path + file_name; - LOG(INFO) << "single replica compaction begin to download file from: " << remote_file_url - << " to: " << local_file_path << ". size(B): " << file_size - << ", timeout(s): " << estimate_timeout; + LOG(INFO) << "single replica compaction begin to download file from: " + << mask_token(remote_file_url) << " to: " << local_file_path + << ". size(B): " << file_size << ", timeout(s): " << estimate_timeout; auto download_cb = [&remote_file_url, estimate_timeout, &local_file_path, file_size](HttpClient* client) { @@ -462,7 +463,8 @@ Status SingleReplicaCompaction::_download_files(DataDir* data_dir, uint64_t local_file_size = std::filesystem::file_size(local_file_path); if (local_file_size != file_size) { LOG(WARNING) << "download file length error" - << ", remote_path=" << remote_file_url << ", file_size=" << file_size + << ", remote_path=" << mask_token(remote_file_url) + << ", file_size=" << file_size << ", local_file_size=" << local_file_size; return Status::InternalError("downloaded file size is not equal"); } diff --git a/be/src/olap/single_replica_compaction.h b/be/src/olap/single_replica_compaction.h index ae013b3748d148..a51954ac93eeab 100644 --- a/be/src/olap/single_replica_compaction.h +++ b/be/src/olap/single_replica_compaction.h @@ -58,10 +58,10 @@ class SingleReplicaCompaction : public Compaction { Status _download_files(DataDir* data_dir, const std::string& remote_url_prefix, const std::string& local_path); Status _release_snapshot(const std::string& ip, int port, const std::string& snapshot_path); - Status _finish_clone(const string& clone_dir, const Version& version); + Status _finish_clone(const std::string& clone_dir, const Version& version); CompactionType _compaction_type; DISALLOW_COPY_AND_ASSIGN(SingleReplicaCompaction); }; -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/olap/task/engine_clone_task.cpp b/be/src/olap/task/engine_clone_task.cpp index 0b077076e574cb..ad9196f5a5ea35 100644 --- a/be/src/olap/task/engine_clone_task.cpp +++ b/be/src/olap/task/engine_clone_task.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -63,6 +62,7 @@ #include "runtime/thread_context.h" #include "util/defer_op.h" #include "util/network_util.h" +#include "util/security.h" #include "util/stopwatch.hpp" #include "util/thrift_rpc_helper.h" #include "util/trace.h" @@ -410,7 +410,7 @@ Status EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir, _clone_req.table_id, _clone_req.partition_id, _clone_req.schema_hash); } else { LOG_WARNING("failed to download snapshot from remote BE") - .tag("url", _mask_token(remote_url_prefix)) + .tag("url", mask_token(remote_url_prefix)) .error(status); } @@ -554,11 +554,11 @@ Status EngineCloneTask::_download_files(DataDir* data_dir, const std::string& re std::string local_file_path = local_path + "/" + file_name; - LOG(INFO) << "clone begin to download file from: " << _mask_token(remote_file_url) + LOG(INFO) << "clone begin to download file from: " << mask_token(remote_file_url) << " to: " << local_file_path << ". size(B): " << file_size << ", timeout(s): " << estimate_timeout; - auto download_cb = [this, &remote_file_url, estimate_timeout, &local_file_path, + auto download_cb = [&remote_file_url, estimate_timeout, &local_file_path, file_size](HttpClient* client) { RETURN_IF_ERROR(client->init(remote_file_url)); client->set_timeout_ms(estimate_timeout * 1000); @@ -574,7 +574,7 @@ Status EngineCloneTask::_download_files(DataDir* data_dir, const std::string& re } if (local_file_size != file_size) { LOG(WARNING) << "download file length error" - << ", remote_path=" << _mask_token(remote_file_url) + << ", remote_path=" << mask_token(remote_file_url) << ", file_size=" << file_size << ", local_file_size=" << local_file_size; return Status::InternalError("downloaded file size is not equal"); @@ -602,7 +602,7 @@ Status EngineCloneTask::_download_files(DataDir* data_dir, const std::string& re /// This method will only be called if tablet already exist in this BE when doing clone. /// This method will do the following things: -/// 1. Linke all files from CLONE dir to tablet dir if file does not exist in tablet dir +/// 1. Link all files from CLONE dir to tablet dir if file does not exist in tablet dir /// 2. Call _finish_xx_clone() to revise the tablet meta. Status EngineCloneTask::_finish_clone(Tablet* tablet, const std::string& clone_dir, int64_t committed_version, bool is_incremental_clone) { @@ -867,9 +867,4 @@ Status EngineCloneTask::_finish_full_clone(Tablet* tablet, // TODO(plat1ko): write cooldown meta to remote if this replica is cooldown replica } -std::string EngineCloneTask::_mask_token(const std::string& str) { - std::regex pattern("token=[\\w|-]+"); - return regex_replace(str, pattern, "token=******"); -} - } // namespace doris diff --git a/be/src/olap/task/engine_clone_task.h b/be/src/olap/task/engine_clone_task.h index 08afc47283a6ce..da8f824d870691 100644 --- a/be/src/olap/task/engine_clone_task.h +++ b/be/src/olap/task/engine_clone_task.h @@ -85,8 +85,6 @@ class EngineCloneTask : public EngineTask { Status _release_snapshot(const std::string& ip, int port, const std::string& snapshot_path); - std::string _mask_token(const std::string& str); - private: const TCloneReq& _clone_req; vector* _tablet_infos; diff --git a/be/src/util/security.h b/be/src/util/security.h new file mode 100644 index 00000000000000..d2201b1b297b70 --- /dev/null +++ b/be/src/util/security.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace doris { + +inline std::string mask_token(const std::string& str) { + std::regex pattern("token=[\\w|-]+"); + return std::regex_replace(str, pattern, "token=******"); +} + +inline std::string mask_token(const char* str) { + std::regex pattern("token=[\\w|-]+"); + return std::regex_replace(str, pattern, "token=******"); +} + +} // namespace doris From 52fd800bf89399e674c2a3975a142aaa7c669f5e Mon Sep 17 00:00:00 2001 From: walter Date: Mon, 18 Nov 2024 15:58:02 +0800 Subject: [PATCH 11/31] [fix](binlog) add modify comment binlog when replaying journal #43827 (#44108) cherry pick from #43827 --- fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 7fdac2ebfaa8c5..bad7011e872917 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -810,6 +810,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_COMMENT: { ModifyCommentOperationLog operation = (ModifyCommentOperationLog) journal.getData(); env.getAlterInstance().replayModifyComment(operation); + env.getBinlogManager().addModifyComment(operation, logId); break; } case OperationType.OP_ALTER_ROUTINE_LOAD_JOB: { From 445813137fea75b57ccad0c8a02d332efdf9df9e Mon Sep 17 00:00:00 2001 From: walter Date: Mon, 18 Nov 2024 16:18:03 +0800 Subject: [PATCH 12/31] [improve](binlog) Reserve ids for binlog type allocation #43769 (#44104) cherry pick from #43769 --- .../apache/doris/binlog/BinlogManager.java | 7 +- gensrc/thrift/FrontendService.thrift | 116 ++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java index 4f67f663879875..ee89c14b40c27f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java @@ -588,8 +588,13 @@ public long read(DataInputStream dis, long checksum) throws IOException { continue; } - // Step 2.2: check if there is in next db Binlogs region long dbId = binlog.getDbId(); + if (binlog.getType().getValue() >= TBinlogType.MIN_UNKNOWN.getValue()) { + LOG.warn("skip unknown binlog, type: {}, db: {}", binlog.getType().getValue(), dbId); + continue; + } + + // Step 2.2: check if there is in next db Binlogs region if (dbId != currentDbId) { // if there is in next db Binlogs region, check and update metadata Database db = Env.getCurrentInternalCatalog().getDbNullable(dbId); diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 6f9242cdd0efa4..a75275bd9174c1 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1030,6 +1030,122 @@ enum TBinlogType { RENAME_TABLE = 14, RENAME_COLUMN = 15, MODIFY_COMMENT = 16, + + // Keep some IDs for allocation so that when new binlog types are added in the + // future, the changes can be picked back to the old versions without breaking + // compatibility. + // + // The code will check the IDs of binlog types, any binlog types whose IDs are + // greater than or equal to MIN_UNKNOWN will be ignored. + // + // For example, before you adding new binlog type MODIFY_XXX: + // MIN_UNKNOWN = 17, + // UNKNOWN_2 = 18, + // UNKNOWN_3 = 19, + // After adding binlog type MODIFY_XXX: + // MODIFY_XXX = 17, + // MIN_UNKNOWN = 18, + // UNKNOWN_3 = 19, + MIN_UNKNOWN = 17, + UNKNOWN_2 = 18, + UNKNOWN_3 = 19, + UNKNOWN_4 = 20, + UNKNOWN_5 = 21, + UNKNOWN_6 = 22, + UNKNOWN_7 = 23, + UNKNOWN_8 = 24, + UNKNOWN_9 = 25, + UNKNOWN_10 = 26, + UNKNOWN_11 = 27, + UNKNOWN_12 = 28, + UNKNOWN_13 = 29, + UNKNOWN_14 = 30, + UNKNOWN_15 = 31, + UNKNOWN_16 = 32, + UNKNOWN_17 = 33, + UNKNOWN_18 = 34, + UNKNOWN_19 = 35, + UNKNOWN_20 = 36, + UNKNOWN_21 = 37, + UNKNOWN_22 = 38, + UNKNOWN_23 = 39, + UNKNOWN_24 = 40, + UNKNOWN_25 = 41, + UNKNOWN_26 = 42, + UNKNOWN_27 = 43, + UNKNOWN_28 = 44, + UNKNOWN_29 = 45, + UNKNOWN_30 = 46, + UNKNOWN_31 = 47, + UNKNOWN_32 = 48, + UNKNOWN_33 = 49, + UNKNOWN_34 = 50, + UNKNOWN_35 = 51, + UNKNOWN_36 = 52, + UNKNOWN_37 = 53, + UNKNOWN_38 = 54, + UNKNOWN_39 = 55, + UNKNOWN_40 = 56, + UNKNOWN_41 = 57, + UNKNOWN_42 = 58, + UNKNOWN_43 = 59, + UNKNOWN_44 = 60, + UNKNOWN_45 = 61, + UNKNOWN_46 = 62, + UNKNOWN_47 = 63, + UNKNOWN_48 = 64, + UNKNOWN_49 = 65, + UNKNOWN_50 = 66, + UNKNOWN_51 = 67, + UNKNOWN_52 = 68, + UNKNOWN_53 = 69, + UNKNOWN_54 = 70, + UNKNOWN_55 = 71, + UNKNOWN_56 = 72, + UNKNOWN_57 = 73, + UNKNOWN_58 = 74, + UNKNOWN_59 = 75, + UNKNOWN_60 = 76, + UNKNOWN_61 = 77, + UNKNOWN_62 = 78, + UNKNOWN_63 = 79, + UNKNOWN_64 = 80, + UNKNOWN_65 = 81, + UNKNOWN_66 = 82, + UNKNOWN_67 = 83, + UNKNOWN_68 = 84, + UNKNOWN_69 = 85, + UNKNOWN_70 = 86, + UNKNOWN_71 = 87, + UNKNOWN_72 = 88, + UNKNOWN_73 = 89, + UNKNOWN_74 = 90, + UNKNOWN_75 = 91, + UNKNOWN_76 = 92, + UNKNOWN_77 = 93, + UNKNOWN_78 = 94, + UNKNOWN_79 = 95, + UNKNOWN_80 = 96, + UNKNOWN_81 = 97, + UNKNOWN_82 = 98, + UNKNOWN_83 = 99, + UNKNOWN_84 = 100, + UNKNOWN_85 = 101, + UNKNOWN_86 = 102, + UNKNOWN_87 = 103, + UNKNOWN_88 = 104, + UNKNOWN_89 = 105, + UNKNOWN_90 = 106, + UNKNOWN_91 = 107, + UNKNOWN_92 = 108, + UNKNOWN_93 = 109, + UNKNOWN_94 = 110, + UNKNOWN_95 = 111, + UNKNOWN_96 = 112, + UNKNOWN_97 = 113, + UNKNOWN_98 = 114, + UNKNOWN_99 = 115, + UNKNOWN_100 = 116, } struct TBinlog { From 04bef055911803b709af285067439957fd26e376 Mon Sep 17 00:00:00 2001 From: John Zhang Date: Mon, 18 Nov 2024 18:47:12 +0800 Subject: [PATCH 13/31] [fix](analyzer) fixed the NullPointerException (#43269) (#44161) cherry-pick: #43269 --- .../apache/doris/qe/cache/CacheAnalyzer.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index 389c5406fe23d4..35ad7ccb189220 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -430,13 +430,18 @@ private List buildCacheTableList() { } public InternalService.PFetchCacheResult getCacheData() throws UserException { - if (parsedStmt instanceof LogicalPlanAdapter) { - cacheMode = innerCheckCacheModeForNereids(0); - } else if (parsedStmt instanceof SelectStmt) { - cacheMode = innerCheckCacheMode(0); - } else if (parsedStmt instanceof SetOperationStmt) { - cacheMode = innerCheckCacheModeSetOperation(0); - } else { + try { + if (parsedStmt instanceof LogicalPlanAdapter) { + cacheMode = innerCheckCacheModeForNereids(0); + } else if (parsedStmt instanceof SelectStmt) { + cacheMode = innerCheckCacheMode(0); + } else if (parsedStmt instanceof SetOperationStmt) { + cacheMode = innerCheckCacheModeSetOperation(0); + } else { + return null; + } + } catch (NullPointerException e) { + LOG.error("getCacheData error", e); return null; } From 6ac54e37ec69f885821cf693b177bbccd36d5a98 Mon Sep 17 00:00:00 2001 From: walter Date: Tue, 19 Nov 2024 10:19:02 +0800 Subject: [PATCH 14/31] [feature](backup) ignore table that not support type when backup, and not report exception (#44200) Cherry-pick #33158 Signed-off-by: nextdreamblue Co-authored-by: xueweizhang --- .../main/java/org/apache/doris/common/Config.java | 6 ++++++ .../java/org/apache/doris/backup/BackupHandler.java | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 7fbb4745ac73f5..dd09db27d94ba1 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1490,6 +1490,12 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true) public static int max_backup_restore_job_num_per_db = 10; + /* + * whether to ignore table that not support type when backup, and not report exception. + */ + @ConfField(mutable = true, masterOnly = true) + public static boolean ignore_backup_not_support_table_type = false; + /** * A internal config, to reduce the restore job size during serialization by compress. * diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index a3fd66692a2928..3015e366c3e11b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -335,6 +335,7 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws // Check if backup objects are valid // This is just a pre-check to avoid most of invalid backup requests. // Also calculate the signature for incremental backup check. + List tblRefsNotSupport = Lists.newArrayList(); for (TableRef tblRef : tblRefs) { String tblName = tblRef.getName().getTbl(); Table tbl = db.getTableOrDdlException(tblName); @@ -342,7 +343,15 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws continue; } if (tbl.getType() != TableType.OLAP) { - ErrorReport.reportDdlException(ErrorCode.ERR_NOT_OLAP_TABLE, tblName); + if (Config.ignore_backup_not_support_table_type) { + LOG.warn("Table '{}' is a {} table, can not backup and ignore it." + + "Only OLAP(Doris)/ODBC/VIEW table can be backed up", + tblName, tbl.getType().toString()); + tblRefsNotSupport.add(tblRef); + continue; + } else { + ErrorReport.reportDdlException(ErrorCode.ERR_NOT_OLAP_TABLE, tblName); + } } OlapTable olapTbl = (OlapTable) tbl; @@ -373,6 +382,8 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws } } + tblRefs.removeAll(tblRefsNotSupport); + // Check if label already be used long repoId = -1; if (repository != null) { From 7c2dfa2f907bda3f3b792e89ecc4f406e86f23dc Mon Sep 17 00:00:00 2001 From: walter Date: Tue, 19 Nov 2024 12:14:31 +0800 Subject: [PATCH 15/31] [refactor](binlog) put recording dropped resource into a seperate method #43938 (#44227) cherry pick from #43938 --- .../org/apache/doris/binlog/DBBinlog.java | 110 +++++++++--------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java index 86cf8085a42b10..e2eef7966be0d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/DBBinlog.java @@ -121,34 +121,7 @@ public void recoverBinlog(TBinlog binlog, boolean dbBinlogEnable) { allBinlogs.add(binlog); binlogSize += BinlogUtils.getApproximateMemoryUsage(binlog); - - if (binlog.getType() == TBinlogType.DROP_PARTITION) { - DropPartitionInfo info = DropPartitionInfo.fromJson(binlog.data); - if (info != null && info.getPartitionId() > 0) { - droppedPartitions.add(Pair.of(info.getPartitionId(), binlog.getCommitSeq())); - } - } else if (binlog.getType() == TBinlogType.DROP_TABLE) { - DropTableRecord record = DropTableRecord.fromJson(binlog.data); - if (record != null && record.getTableId() > 0) { - droppedTables.add(Pair.of(record.getTableId(), binlog.getCommitSeq())); - } - } else if (binlog.getType() == TBinlogType.ALTER_JOB) { - AlterJobRecord record = AlterJobRecord.fromJson(binlog.data); - if (record != null && record.isSchemaChangeJob() && record.isJobFinished()) { - for (Long indexId : record.getOriginIndexIdList()) { - if (indexId != null && indexId > 0) { - droppedIndexes.add(Pair.of(indexId, binlog.getCommitSeq())); - } - } - } - } else if (binlog.getType() == TBinlogType.TRUNCATE_TABLE) { - TruncateTableRecord record = TruncateTableRecord.fromJson(binlog.data); - if (record != null) { - for (long partitionId : record.getOldPartitionIds()) { - droppedPartitions.add(Pair.of(partitionId, binlog.getCommitSeq())); - } - } - } + recordDroppedResources(binlog); if (tableIds == null) { return; @@ -202,31 +175,7 @@ public void addBinlog(TBinlog binlog, Object raw) { return; } - if (binlog.getType() == TBinlogType.DROP_PARTITION && raw instanceof DropPartitionInfo) { - long partitionId = ((DropPartitionInfo) raw).getPartitionId(); - if (partitionId > 0) { - droppedPartitions.add(Pair.of(partitionId, binlog.getCommitSeq())); - } - } else if (binlog.getType() == TBinlogType.DROP_TABLE && raw instanceof DropTableRecord) { - long tableId = ((DropTableRecord) raw).getTableId(); - if (tableId > 0) { - droppedTables.add(Pair.of(tableId, binlog.getCommitSeq())); - } - } else if (binlog.getType() == TBinlogType.ALTER_JOB && raw instanceof AlterJobRecord) { - AlterJobRecord alterJobRecord = (AlterJobRecord) raw; - if (alterJobRecord.isJobFinished() && alterJobRecord.isSchemaChangeJob()) { - for (Long indexId : alterJobRecord.getOriginIndexIdList()) { - if (indexId != null && indexId > 0) { - droppedIndexes.add(Pair.of(indexId, binlog.getCommitSeq())); - } - } - } - } else if (binlog.getType() == TBinlogType.TRUNCATE_TABLE && raw instanceof TruncateTableRecord) { - TruncateTableRecord truncateTableRecord = (TruncateTableRecord) raw; - for (long partitionId : truncateTableRecord.getOldPartitionIds()) { - droppedPartitions.add(Pair.of(partitionId, binlog.getCommitSeq())); - } - } + recordDroppedResources(binlog, raw); switch (binlog.getType()) { case CREATE_TABLE: @@ -670,4 +619,59 @@ public void getBinlogInfo(BaseProcResult result) { lock.readLock().unlock(); } } + + private void recordDroppedResources(TBinlog binlog) { + recordDroppedResources(binlog, null); + } + + // A method to record the dropped tables, indexes, and partitions. + private void recordDroppedResources(TBinlog binlog, Object raw) { + if (raw == null) { + switch (binlog.getType()) { + case DROP_PARTITION: + raw = DropPartitionInfo.fromJson(binlog.data); + break; + case DROP_TABLE: + raw = DropTableRecord.fromJson(binlog.data); + break; + case ALTER_JOB: + raw = AlterJobRecord.fromJson(binlog.data); + break; + case TRUNCATE_TABLE: + raw = TruncateTableRecord.fromJson(binlog.data); + break; + default: + break; + } + if (raw == null) { + return; + } + } + + if (binlog.getType() == TBinlogType.DROP_PARTITION && raw instanceof DropPartitionInfo) { + long partitionId = ((DropPartitionInfo) raw).getPartitionId(); + if (partitionId > 0) { + droppedPartitions.add(Pair.of(partitionId, binlog.getCommitSeq())); + } + } else if (binlog.getType() == TBinlogType.DROP_TABLE && raw instanceof DropTableRecord) { + long tableId = ((DropTableRecord) raw).getTableId(); + if (tableId > 0) { + droppedTables.add(Pair.of(tableId, binlog.getCommitSeq())); + } + } else if (binlog.getType() == TBinlogType.ALTER_JOB && raw instanceof AlterJobRecord) { + AlterJobRecord alterJobRecord = (AlterJobRecord) raw; + if (alterJobRecord.isJobFinished() && alterJobRecord.isSchemaChangeJob()) { + for (Long indexId : alterJobRecord.getOriginIndexIdList()) { + if (indexId != null && indexId > 0) { + droppedIndexes.add(Pair.of(indexId, binlog.getCommitSeq())); + } + } + } + } else if (binlog.getType() == TBinlogType.TRUNCATE_TABLE && raw instanceof TruncateTableRecord) { + TruncateTableRecord truncateTableRecord = (TruncateTableRecord) raw; + for (long partitionId : truncateTableRecord.getOldPartitionIds()) { + droppedPartitions.add(Pair.of(partitionId, binlog.getCommitSeq())); + } + } + } } From f5d29933c947aad1d34d9200dfc32a4965a744c1 Mon Sep 17 00:00:00 2001 From: walter Date: Tue, 19 Nov 2024 12:14:54 +0800 Subject: [PATCH 16/31] [improve](backup) Add the commit seq at the backup job level. #44049 (#44111) cherry pick from #44049 --- .../apache/doris/backup/BackupHandler.java | 43 ++++++++++++++----- .../org/apache/doris/backup/BackupJob.java | 34 ++++++++++++--- .../org/apache/doris/backup/Snapshot.java | 11 ++++- .../org/apache/doris/catalog/Database.java | 4 ++ .../org/apache/doris/persist/BarrierLog.java | 5 +++ .../doris/service/FrontendServiceImpl.java | 6 ++- .../apache/doris/backup/BackupJobTest.java | 4 +- gensrc/thrift/FrontendService.thrift | 1 + 8 files changed, 85 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java index 3015e366c3e11b..61d1e9e5f1347e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java @@ -49,6 +49,7 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.fs.remote.RemoteFileSystem; +import org.apache.doris.persist.BarrierLog; import org.apache.doris.task.DirMoveTask; import org.apache.doris.task.DownloadTask; import org.apache.doris.task.SnapshotTask; @@ -308,20 +309,32 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws + " is read only"); } - // Determine the tables to be backed up + long commitSeq = 0; Set tableNames = Sets.newHashSet(); AbstractBackupTableRefClause abstractBackupTableRefClause = stmt.getAbstractBackupTableRefClause(); - if (abstractBackupTableRefClause == null) { - tableNames = db.getTableNamesWithLock(); - } else if (abstractBackupTableRefClause.isExclude()) { - tableNames = db.getTableNamesWithLock(); - for (TableRef tableRef : abstractBackupTableRefClause.getTableRefList()) { - if (!tableNames.remove(tableRef.getName().getTbl())) { - LOG.info("exclude table " + tableRef.getName().getTbl() - + " of backup stmt is not exists in db " + db.getFullName()); + + // Obtain the snapshot commit seq, any creating table binlog will be visible. + db.readLock(); + try { + BarrierLog log = new BarrierLog(db.getId(), db.getFullName()); + commitSeq = env.getEditLog().logBarrier(log); + + // Determine the tables to be backed up + if (abstractBackupTableRefClause == null) { + tableNames = db.getTableNames(); + } else if (abstractBackupTableRefClause.isExclude()) { + tableNames = db.getTableNames(); + for (TableRef tableRef : abstractBackupTableRefClause.getTableRefList()) { + if (!tableNames.remove(tableRef.getName().getTbl())) { + LOG.info("exclude table " + tableRef.getName().getTbl() + + " of backup stmt is not exists in db " + db.getFullName()); + } } } + } finally { + db.readUnlock(); } + List tblRefs = Lists.newArrayList(); if (abstractBackupTableRefClause != null && !abstractBackupTableRefClause.isExclude()) { tblRefs = abstractBackupTableRefClause.getTableRefList(); @@ -339,6 +352,14 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws for (TableRef tblRef : tblRefs) { String tblName = tblRef.getName().getTbl(); Table tbl = db.getTableOrDdlException(tblName); + + // filter the table types which are not supported by local backup. + if (repository == null && tbl.getType() != TableType.OLAP + && tbl.getType() != TableType.VIEW && tbl.getType() != TableType.MATERIALIZED_VIEW) { + tblRefsNotSupport.add(tblRef); + continue; + } + if (tbl.getType() == TableType.VIEW || tbl.getType() == TableType.ODBC) { continue; } @@ -385,7 +406,7 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws tblRefs.removeAll(tblRefsNotSupport); // Check if label already be used - long repoId = -1; + long repoId = Repository.KEEP_ON_LOCAL_REPO_ID; if (repository != null) { List existSnapshotNames = Lists.newArrayList(); Status st = repository.listSnapshots(existSnapshotNames); @@ -407,7 +428,7 @@ private void backup(Repository repository, Database db, BackupStmt stmt) throws // Create a backup job BackupJob backupJob = new BackupJob(stmt.getLabel(), db.getId(), ClusterNamespace.getNameFromFullName(db.getFullName()), - tblRefs, stmt.getTimeoutMs(), stmt.getContent(), env, repoId); + tblRefs, stmt.getTimeoutMs(), stmt.getContent(), env, repoId, commitSeq); // write log env.getEditLog().logBackupJob(backupJob); diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java index 8a5043ebe7f61a..59365a24931407 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java @@ -36,6 +36,7 @@ import org.apache.doris.catalog.View; import org.apache.doris.common.Config; import org.apache.doris.common.io.Text; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.property.S3ClientBEProperties; import org.apache.doris.persist.BarrierLog; @@ -85,6 +86,7 @@ public class BackupJob extends AbstractJob { private static final Logger LOG = LogManager.getLogger(BackupJob.class); private static final String TABLE_COMMIT_SEQ_PREFIX = "table_commit_seq:"; + private static final String SNAPSHOT_COMMIT_SEQ = "commit_seq"; public enum BackupJobState { PENDING, // Job is newly created. Send snapshot tasks and save copied meta info, then transfer to SNAPSHOTING @@ -123,6 +125,8 @@ public enum BackupJobState { // backup properties && table commit seq with table id private Map properties = Maps.newHashMap(); + private long commitSeq = 0; + public BackupJob() { super(JobType.BACKUP); } @@ -133,11 +137,13 @@ public BackupJob(JobType jobType) { } public BackupJob(String label, long dbId, String dbName, List tableRefs, long timeoutMs, - BackupContent content, Env env, long repoId) { + BackupContent content, Env env, long repoId, long commitSeq) { super(JobType.BACKUP, label, dbId, dbName, timeoutMs, env, repoId); this.tableRefs = tableRefs; this.state = BackupJobState.PENDING; + this.commitSeq = commitSeq; properties.put(BackupStmt.PROP_CONTENT, content.name()); + properties.put(SNAPSHOT_COMMIT_SEQ, String.valueOf(commitSeq)); } public BackupJobState getState() { @@ -237,7 +243,7 @@ public synchronized boolean finishTabletSnapshotTask(SnapshotTask task, TFinishT if (request.getTaskStatus().getStatusCode() == TStatusCode.TABLET_MISSING && !tryNewTabletSnapshotTask(task)) { status = new Status(ErrCode.NOT_FOUND, - "make snapshot failed, failed to ge tablet, table will be droped or truncated"); + "make snapshot failed, failed to ge tablet, table will be dropped or truncated"); cancelInternal(); } @@ -379,6 +385,14 @@ public synchronized void run() { LOG.debug("run backup job: {}", this); + if (state == BackupJobState.PENDING) { + String pausedLabel = DebugPointUtil.getDebugParamOrDefault("FE.PAUSE_PENDING_BACKUP_JOB", ""); + if (!pausedLabel.isEmpty() && label.startsWith(pausedLabel)) { + LOG.info("pause pending backup job by debug point: {}", this); + return; + } + } + // run job base on current state switch (state) { case PENDING: @@ -526,7 +540,7 @@ private void checkOlapTable(OlapTable olapTable, TableRef backupTableRef) { private void prepareSnapshotTaskForOlapTableWithoutLock(Database db, OlapTable olapTable, TableRef backupTableRef, AgentBatchTask batchTask) { - // Add barrier editolog for barrier commit seq + // Add barrier editlog for barrier commit seq long dbId = db.getId(); String dbName = db.getFullName(); long tableId = olapTable.getId(); @@ -656,13 +670,11 @@ private void removeUnsupportProperties(OlapTable tbl) { private void waitingAllSnapshotsFinished() { if (unfinishedTaskIds.isEmpty()) { - if (env.getEditLog().exceedMaxJournalSize(this)) { status = new Status(ErrCode.COMMON_ERROR, "backupJob is too large "); return; } - snapshotFinishedTime = System.currentTimeMillis(); state = BackupJobState.UPLOAD_SNAPSHOT; @@ -972,6 +984,10 @@ public boolean isLocalSnapshot() { return repoId == Repository.KEEP_ON_LOCAL_REPO_ID; } + public long getCommitSeq() { + return commitSeq; + } + // read meta and job info bytes from disk, and return the snapshot public synchronized Snapshot getSnapshot() { if (state != BackupJobState.FINISHED || repoId != Repository.KEEP_ON_LOCAL_REPO_ID) { @@ -981,7 +997,7 @@ public synchronized Snapshot getSnapshot() { // Avoid loading expired meta. long expiredAt = createTime + timeoutMs; if (System.currentTimeMillis() >= expiredAt) { - return new Snapshot(label, new byte[0], new byte[0], expiredAt); + return new Snapshot(label, new byte[0], new byte[0], expiredAt, commitSeq); } try { @@ -989,7 +1005,7 @@ public synchronized Snapshot getSnapshot() { File jobInfoFile = new File(localJobInfoFilePath); byte[] metaInfoBytes = Files.readAllBytes(metaInfoFile.toPath()); byte[] jobInfoBytes = Files.readAllBytes(jobInfoFile.toPath()); - return new Snapshot(label, metaInfoBytes, jobInfoBytes, expiredAt); + return new Snapshot(label, metaInfoBytes, jobInfoBytes, expiredAt, commitSeq); } catch (IOException e) { LOG.warn("failed to load meta info and job info file, meta info file {}, job info file {}: ", localMetaInfoFilePath, localJobInfoFilePath, e); @@ -1182,6 +1198,10 @@ public void readOthers(DataInput in) throws IOException { String value = Text.readString(in); properties.put(key, value); } + + if (properties.containsKey(SNAPSHOT_COMMIT_SEQ)) { + commitSeq = Long.parseLong(properties.get(SNAPSHOT_COMMIT_SEQ)); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java b/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java index c4c93548177ca5..a9f734dbc99220 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/Snapshot.java @@ -34,14 +34,18 @@ public class Snapshot { @SerializedName(value = "expired_at") private long expiredAt = 0; + @SerializedName(value = "commitSeq") + private long commitSeq = 0; + public Snapshot() { } - public Snapshot(String label, byte[] meta, byte[] jobInfo, long expiredAt) { + public Snapshot(String label, byte[] meta, byte[] jobInfo, long expiredAt, long commitSeq) { this.label = label; this.meta = meta; this.jobInfo = jobInfo; this.expiredAt = expiredAt; + this.commitSeq = commitSeq; } public byte[] getMeta() { @@ -60,6 +64,10 @@ public boolean isExpired() { return System.currentTimeMillis() > expiredAt; } + public long getCommitSeq() { + return commitSeq; + } + public String toJson() { return GsonUtils.GSON.toJson(this); } @@ -71,6 +79,7 @@ public String toString() { + ", meta=" + meta + ", jobInfo=" + jobInfo + ", expiredAt=" + expiredAt + + ", commitSeq=" + commitSeq + '}'; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java index b218109383230d..b81068d475c2de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java @@ -518,6 +518,10 @@ public Set getTableNamesWithLock() { } } + public Set getTableNames() { + return new HashSet<>(this.nameToTable.keySet()); + } + /** * This is a thread-safe method when nameToTable is a concurrent hash map */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/BarrierLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/BarrierLog.java index 2b4245b290c850..4a9ce13e03b3ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/BarrierLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/BarrierLog.java @@ -46,6 +46,11 @@ public class BarrierLog implements Writable { public BarrierLog() { } + public BarrierLog(long dbId, String dbName) { + this.dbId = dbId; + this.dbName = dbName; + } + public BarrierLog(long dbId, String dbName, long tableId, String tableName) { this.dbId = dbId; this.dbName = dbName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 8c521cc73b0b52..e67534d302b9c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -2820,9 +2820,10 @@ private TGetSnapshotResult getSnapshotImpl(TGetSnapshotRequest request, String c byte[] meta = snapshot.getMeta(); byte[] jobInfo = snapshot.getJobInfo(); long expiredAt = snapshot.getExpiredAt(); + long commitSeq = snapshot.getCommitSeq(); - LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info size: {}, expired at: {}", - label, meta.length, jobInfo.length, expiredAt); + LOG.info("get snapshot info, snapshot: {}, meta size: {}, job info size: {}, " + + "expired at: {}, commit seq: {}", label, meta.length, jobInfo.length, expiredAt, commitSeq); if (request.isEnableCompress()) { meta = GZIPUtils.compress(meta); jobInfo = GZIPUtils.compress(jobInfo); @@ -2835,6 +2836,7 @@ private TGetSnapshotResult getSnapshotImpl(TGetSnapshotRequest request, String c result.setMeta(meta); result.setJobInfo(jobInfo); result.setExpiredAt(expiredAt); + result.setCommitSeq(commitSeq); } return result; diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java index 4e0eecda1fa7d3..dd9cb0752934b9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/BackupJobTest.java @@ -212,7 +212,7 @@ Status getBrokerAddress(Long beId, Env env, List brokerAddrs) { new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, UnitTestUtil.DB_NAME, UnitTestUtil.TABLE_NAME), null)); job = new BackupJob("label", dbId, UnitTestUtil.DB_NAME, tableRefs, 13600 * 1000, BackupStmt.BackupContent.ALL, - env, repo.getId()); + env, repo.getId(), 0); } @Test @@ -348,7 +348,7 @@ public void testRunAbnormal() { new TableRef(new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, UnitTestUtil.DB_NAME, "unknown_tbl"), null)); job = new BackupJob("label", dbId, UnitTestUtil.DB_NAME, tableRefs, 13600 * 1000, BackupStmt.BackupContent.ALL, - env, repo.getId()); + env, repo.getId(), 0); job.run(); Assert.assertEquals(Status.ErrCode.NOT_FOUND, job.getStatus().getErrCode()); Assert.assertEquals(BackupJobState.CANCELLED, job.getState()); diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index a75275bd9174c1..b58f8a42e8924d 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1204,6 +1204,7 @@ struct TGetSnapshotResult { 4: optional Types.TNetworkAddress master_address 5: optional bool compressed; 6: optional i64 expiredAt; // in millis + 7: optional i64 commit_seq; } struct TTableRef { From a8a1dbebc407fede312d894aed2847c83023a118 Mon Sep 17 00:00:00 2001 From: zhangyuan Date: Tue, 19 Nov 2024 19:03:37 +0800 Subject: [PATCH 17/31] [bugfix](hudi) catch exception when getting hudi partition (#35027) (#41342) bp https://github.com/apache/doris/pull/35027 Hudi use a thread pool to get files for each partition. And use a countdown latch to wait all threads finish. But if the thread throw exception, the countdown latch will not be counted down, and thread will be blocked. --- .../planner/external/hudi/HudiScanNode.java | 122 ++++++++++-------- 1 file changed, 67 insertions(+), 55 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java index 803dadae03d5d4..ab417a19cf725d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/hudi/HudiScanNode.java @@ -72,6 +72,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; public class HudiScanNode extends HiveScanNode { @@ -272,73 +273,84 @@ public List getSplits() throws UserException { .getExtMetaCacheMgr().getHudiPartitionProcess(hmsTable.getCatalog())).getExecutor(); List splits = Collections.synchronizedList(new ArrayList<>()); CountDownLatch countDownLatch = new CountDownLatch(partitions.size()); + AtomicReference throwable = new AtomicReference<>(); partitions.forEach(partition -> executor.execute(() -> { - String globPath; - String partitionName = ""; - if (partition.isDummyPartition()) { - globPath = hudiClient.getBasePathV2().toString() + "/*"; - } else { - partitionName = FSUtils.getRelativePartitionPath(hudiClient.getBasePathV2(), - new Path(partition.getPath())); - globPath = String.format("%s/%s/*", hudiClient.getBasePathV2().toString(), partitionName); - } - List statuses; try { - statuses = FSUtils.getGlobStatusExcludingMetaFolder(hudiClient.getRawFs(), - new Path(globPath)); - } catch (IOException e) { - throw new RuntimeException("Failed to get hudi file statuses on path: " + globPath, e); - } - HoodieTableFileSystemView fileSystemView = new HoodieTableFileSystemView(hudiClient, - timeline, statuses.toArray(new FileStatus[0])); + String globPath; + String partitionName = ""; + if (partition.isDummyPartition()) { + globPath = hudiClient.getBasePathV2().toString() + "/*"; + } else { + partitionName = FSUtils.getRelativePartitionPath(hudiClient.getBasePathV2(), + new Path(partition.getPath())); + globPath = String.format("%s/%s/*", hudiClient.getBasePathV2().toString(), partitionName); + } + List statuses; + try { + statuses = FSUtils.getGlobStatusExcludingMetaFolder(hudiClient.getRawFs(), + new Path(globPath)); + } catch (IOException e) { + throw new RuntimeException("Failed to get hudi file statuses on path: " + globPath, e); + } + HoodieTableFileSystemView fileSystemView = new HoodieTableFileSystemView(hudiClient, + timeline, statuses.toArray(new FileStatus[0])); - if (isCowOrRoTable) { - fileSystemView.getLatestBaseFilesBeforeOrOn(partitionName, queryInstant).forEach(baseFile -> { - noLogsSplitNum.incrementAndGet(); - String filePath = baseFile.getPath(); - long fileSize = baseFile.getFileSize(); - // Need add hdfs host to location - LocationPath locationPath = new LocationPath(filePath, hmsTable.getCatalogProperties()); - Path splitFilePath = locationPath.toScanRangeLocation(); - splits.add(new FileSplit(splitFilePath, 0, fileSize, fileSize, - new String[0], partition.getPartitionValues())); - }); - } else { - fileSystemView.getLatestMergedFileSlicesBeforeOrOn(partitionName, queryInstant).forEach(fileSlice -> { - Optional baseFile = fileSlice.getBaseFile().toJavaOptional(); - String filePath = baseFile.map(BaseFile::getPath).orElse(""); - long fileSize = baseFile.map(BaseFile::getFileSize).orElse(0L); + if (isCowOrRoTable) { + fileSystemView.getLatestBaseFilesBeforeOrOn(partitionName, queryInstant) + .forEach(baseFile -> { + noLogsSplitNum.incrementAndGet(); + String filePath = baseFile.getPath(); + long fileSize = baseFile.getFileSize(); + // Need add hdfs host to location + LocationPath locationPath = new LocationPath(filePath, hmsTable.getCatalogProperties()); + Path splitFilePath = locationPath.toScanRangeLocation(); + splits.add(new FileSplit(splitFilePath, 0, fileSize, fileSize, + new String[0], partition.getPartitionValues())); + }); + } else { + fileSystemView.getLatestMergedFileSlicesBeforeOrOn(partitionName, queryInstant) + .forEach(fileSlice -> { + Optional baseFile = fileSlice.getBaseFile().toJavaOptional(); + String filePath = baseFile.map(BaseFile::getPath).orElse(""); + long fileSize = baseFile.map(BaseFile::getFileSize).orElse(0L); - List logs = fileSlice.getLogFiles().map(HoodieLogFile::getPath) - .map(Path::toString) - .collect(Collectors.toList()); - if (logs.isEmpty()) { - noLogsSplitNum.incrementAndGet(); - } + List logs = fileSlice.getLogFiles().map(HoodieLogFile::getPath) + .map(Path::toString) + .collect(Collectors.toList()); + if (logs.isEmpty()) { + noLogsSplitNum.incrementAndGet(); + } - // no base file, use log file to parse file type - String agencyPath = filePath.isEmpty() ? logs.get(0) : filePath; - HudiSplit split = new HudiSplit(new Path(agencyPath), 0, fileSize, fileSize, - new String[0], partition.getPartitionValues()); - split.setTableFormatType(TableFormatType.HUDI); - split.setDataFilePath(filePath); - split.setHudiDeltaLogs(logs); - split.setInputFormat(inputFormat); - split.setSerde(serdeLib); - split.setBasePath(basePath); - split.setHudiColumnNames(columnNames); - split.setHudiColumnTypes(columnTypes); - split.setInstantTime(queryInstant); - splits.add(split); - }); + // no base file, use log file to parse file type + String agencyPath = filePath.isEmpty() ? logs.get(0) : filePath; + HudiSplit split = new HudiSplit(new Path(agencyPath), 0, fileSize, fileSize, + new String[0], partition.getPartitionValues()); + split.setTableFormatType(TableFormatType.HUDI); + split.setDataFilePath(filePath); + split.setHudiDeltaLogs(logs); + split.setInputFormat(inputFormat); + split.setSerde(serdeLib); + split.setBasePath(basePath); + split.setHudiColumnNames(columnNames); + split.setHudiColumnTypes(columnTypes); + split.setInstantTime(queryInstant); + splits.add(split); + }); + } + } catch (Throwable t) { + throwable.set(t); + } finally { + countDownLatch.countDown(); } - countDownLatch.countDown(); })); try { countDownLatch.await(); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } + if (throwable.get() != null) { + throw new RuntimeException(throwable.get().getMessage(), throwable.get()); + } return splits; } From cab615f8023e12fd3239a7f18b14fe444965e520 Mon Sep 17 00:00:00 2001 From: walter Date: Wed, 20 Nov 2024 14:09:56 +0800 Subject: [PATCH 18/31] [opt](binlog) Support drop view binlog #39781 (#44114) cherry pick from #39781 Co-authored-by: smallx --- .../doris/alter/MaterializedViewHandler.java | 2 +- .../java/org/apache/doris/backup/RestoreJob.java | 6 ++++-- .../org/apache/doris/binlog/DropTableRecord.java | 7 ++++++- .../apache/doris/datasource/InternalCatalog.java | 15 ++++++++------- .../java/org/apache/doris/persist/DropInfo.java | 16 ++++++++++++---- .../java/org/apache/doris/persist/EditLog.java | 2 +- .../doris/persist/DropAndRecoverInfoTest.java | 10 +++++----- 7 files changed, 37 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index 9e997da1c0a8db..52bcbb71ab83c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -935,7 +935,7 @@ public void processDropMaterializedView(DropMaterializedViewStmt dropMaterialize // Step3: log drop mv operation EditLog editLog = Env.getCurrentEnv().getEditLog(); editLog.logDropRollup( - new DropInfo(db.getId(), olapTable.getId(), olapTable.getName(), mvIndexId, false, 0)); + new DropInfo(db.getId(), olapTable.getId(), olapTable.getName(), mvIndexId, false, false, 0)); LOG.info("finished drop materialized view [{}] in table [{}]", mvName, olapTable.getName()); } catch (MetaNotFoundException e) { if (dropMaterializedViewStmt.isIfExists()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 1db289dbaa9cb7..709df8dfe571a3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -2069,13 +2069,15 @@ private Status dropAllNonRestoredTableAndPartitions(Database db) { } else if (isCleanTables) { // otherwise drop the entire table. LOG.info("drop non restored table {}, table id: {}. {}", tableName, tableId, this); + boolean isView = false; boolean isForceDrop = false; // move this table into recyclebin. - env.getInternalCatalog().dropTableWithoutCheck(db, table, isForceDrop); + env.getInternalCatalog().dropTableWithoutCheck(db, table, isView, isForceDrop); } } else if (tableType == TableType.VIEW && isCleanTables && !restoredViews.contains(tableName)) { LOG.info("drop non restored view {}, table id: {}. {}", tableName, tableId, this); + boolean isView = false; boolean isForceDrop = false; // move this view into recyclebin. - env.getInternalCatalog().dropTableWithoutCheck(db, table, isForceDrop); + env.getInternalCatalog().dropTableWithoutCheck(db, table, isView, isForceDrop); } } return Status.OK; diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/DropTableRecord.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/DropTableRecord.java index 4417edeb97372d..c998f2e73fee42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/DropTableRecord.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/DropTableRecord.java @@ -31,6 +31,8 @@ public class DropTableRecord { private long tableId; @SerializedName(value = "tableName") private String tableName; + @SerializedName(value = "isView") + private boolean isView = false; @SerializedName(value = "rawSql") private String rawSql; @@ -39,7 +41,10 @@ public DropTableRecord(long commitSeq, DropInfo info) { this.dbId = info.getDbId(); this.tableId = info.getTableId(); this.tableName = info.getTableName(); - this.rawSql = String.format("DROP TABLE IF EXISTS `%s`", this.tableName); + this.isView = info.isView(); + this.rawSql = info.isView() + ? String.format("DROP VIEW IF EXISTS `%s`", this.tableName) + : String.format("DROP TABLE IF EXISTS `%s`", this.tableName); } public long getCommitSeq() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index e78ef3334f123d..a65c82f721e674 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -916,23 +916,24 @@ public void dropTable(DropTableStmt stmt) throws DdlException { } } - dropTableInternal(db, table, stmt.isForceDrop()); + dropTableInternal(db, table, stmt.isView(), stmt.isForceDrop()); } catch (UserException e) { throw new DdlException(e.getMessage(), e.getMysqlErrorCode()); } finally { db.writeUnlock(); } - LOG.info("finished dropping table: {} from db: {}, is force: {}", tableName, dbName, stmt.isForceDrop()); + LOG.info("finished dropping table: {} from db: {}, is view: {}, is force: {}", + tableName, dbName, stmt.isView(), stmt.isForceDrop()); } // drop table without any check. - public void dropTableWithoutCheck(Database db, Table table, boolean forceDrop) throws DdlException { + public void dropTableWithoutCheck(Database db, Table table, boolean isView, boolean forceDrop) throws DdlException { if (!db.writeLockIfExist()) { return; } try { LOG.info("drop table {} without check, force: {}", table.getQualifiedName(), forceDrop); - dropTableInternal(db, table, forceDrop); + dropTableInternal(db, table, isView, forceDrop); } catch (Exception e) { LOG.warn("drop table without check", e); throw e; @@ -942,7 +943,7 @@ public void dropTableWithoutCheck(Database db, Table table, boolean forceDrop) t } // Drop a table, the db lock must hold. - private void dropTableInternal(Database db, Table table, boolean forceDrop) throws DdlException { + private void dropTableInternal(Database db, Table table, boolean isView, boolean forceDrop) throws DdlException { table.writeLock(); String tableName = table.getName(); long recycleTime = 0; @@ -955,7 +956,7 @@ private void dropTableInternal(Database db, Table table, boolean forceDrop) thro table.writeUnlock(); } - DropInfo info = new DropInfo(db.getId(), table.getId(), tableName, -1L, forceDrop, recycleTime); + DropInfo info = new DropInfo(db.getId(), table.getId(), tableName, -1L, isView, forceDrop, recycleTime); Env.getCurrentEnv().getEditLog().logDropTable(info); Env.getCurrentEnv().getQueryStats().clear(Env.getCurrentEnv().getCurrentCatalog().getId(), db.getId(), table.getId()); @@ -2732,7 +2733,7 @@ private boolean createOlapTable(Database db, CreateTableStmt stmt) throws UserEx try { dropTable(db, tableId, true, false, 0L); if (hadLogEditCreateTable) { - DropInfo info = new DropInfo(db.getId(), tableId, olapTable.getName(), -1L, true, 0L); + DropInfo info = new DropInfo(db.getId(), tableId, olapTable.getName(), -1L, false, true, 0L); Env.getCurrentEnv().getEditLog().logDropTable(info); } } catch (Exception ex) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java index b30522e942592b..461f3ddd67d5a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java @@ -38,6 +38,8 @@ public class DropInfo implements Writable { private String tableName; // not used in equals and hashCode @SerializedName(value = "indexId") private long indexId; + @SerializedName(value = "isView") + private boolean isView = false; @SerializedName(value = "forceDrop") private boolean forceDrop = false; @SerializedName(value = "recycleTime") @@ -46,11 +48,13 @@ public class DropInfo implements Writable { public DropInfo() { } - public DropInfo(long dbId, long tableId, String tableName, long indexId, boolean forceDrop, long recycleTime) { + public DropInfo(long dbId, long tableId, String tableName, long indexId, boolean isView, boolean forceDrop, + long recycleTime) { this.dbId = dbId; this.tableId = tableId; this.tableName = tableName; this.indexId = indexId; + this.isView = isView; this.forceDrop = forceDrop; this.recycleTime = recycleTime; } @@ -71,12 +75,16 @@ public long getIndexId() { return this.indexId; } + public boolean isView() { + return this.isView; + } + public boolean isForceDrop() { - return forceDrop; + return this.forceDrop; } public Long getRecycleTime() { - return recycleTime; + return this.recycleTime; } @Override @@ -119,7 +127,7 @@ public boolean equals(Object obj) { DropInfo info = (DropInfo) obj; return (dbId == info.dbId) && (tableId == info.tableId) && (indexId == info.indexId) - && (forceDrop == info.forceDrop) && (recycleTime == info.recycleTime); + && (isView == info.isView) && (forceDrop == info.forceDrop) && (recycleTime == info.recycleTime); } public String toJson() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index bad7011e872917..54c12dfd0463e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -336,7 +336,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { for (long indexId : batchDropInfo.getIndexIdSet()) { env.getMaterializedViewHandler().replayDropRollup( new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(), - batchDropInfo.getTableName(), indexId, false, 0), + batchDropInfo.getTableName(), indexId, false, false, 0), env); } break; diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java index bdaab002c53180..88aa22ded22e5e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java @@ -44,7 +44,7 @@ public void testDropInfoSerialization() throws Exception { DropInfo info1 = new DropInfo(); info1.write(dos); - DropInfo info2 = new DropInfo(1, 2, "t2", -1, true, 0); + DropInfo info2 = new DropInfo(1, 2, "t2", -1, false, true, 0); info2.write(dos); dos.flush(); @@ -65,10 +65,10 @@ public void testDropInfoSerialization() throws Exception { Assert.assertEquals(rInfo2, rInfo2); Assert.assertNotEquals(rInfo2, this); - Assert.assertNotEquals(info2, new DropInfo(0, 2, "t2", -1L, true, 0)); - Assert.assertNotEquals(info2, new DropInfo(1, 0, "t0", -1L, true, 0)); - Assert.assertNotEquals(info2, new DropInfo(1, 2, "t2", -1L, false, 0)); - Assert.assertEquals(info2, new DropInfo(1, 2, "t2", -1L, true, 0)); + Assert.assertNotEquals(info2, new DropInfo(0, 2, "t2", -1L, false, true, 0)); + Assert.assertNotEquals(info2, new DropInfo(1, 0, "t0", -1L, false, true, 0)); + Assert.assertNotEquals(info2, new DropInfo(1, 2, "t2", -1L, false, false, 0)); + Assert.assertEquals(info2, new DropInfo(1, 2, "t2", -1L, false, true, 0)); // 3. delete files dis.close(); From f7dda9e3f67dd24d6d39b3efbc411e9cfb2b88e0 Mon Sep 17 00:00:00 2001 From: starocean999 Date: Thu, 21 Nov 2024 11:50:10 +0800 Subject: [PATCH 19/31] [fix](nereids)should set isForwardedToMaster and redirectStatus to null before fallback to legacy planner (#44333) pick from master https://github.com/apache/doris/pull/44163 --- fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index e4cb541744cdfe..0e09640ba2c8f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -515,6 +515,8 @@ public void execute(TUniqueId queryId) throws Exception { LOG.debug("fall back to legacy planner on statement:\n{}", originStmt.originStmt); parsedStmt = null; planner = null; + isForwardedToMaster = null; + redirectStatus = null; // Attention: currently exception from nereids does not mean an Exception to user terminal // unless user does not allow fallback to lagency planner. But state of query // has already been set to Error in this case, it will have some side effect on profile result From e8d29186007a6ed8a6fb8c47cbe55a9eb68727ff Mon Sep 17 00:00:00 2001 From: walter Date: Mon, 25 Nov 2024 15:46:50 +0800 Subject: [PATCH 20/31] [opt] (binlog) Support Modify ViewDef binlog #41167 (#44520) cherry pick from #41167 Co-authored-by: yanmingfu <133083714+xiaoming12306@users.noreply.github.com> Co-authored-by: yanmingfu --- .../apache/doris/binlog/BinlogManager.java | 65 +++++++++++-------- .../apache/doris/persist/AlterViewInfo.java | 4 ++ .../org/apache/doris/persist/EditLog.java | 7 +- gensrc/thrift/FrontendService.thrift | 4 +- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java index ee89c14b40c27f..d4be686a14ba31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/binlog/BinlogManager.java @@ -25,6 +25,7 @@ import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.proc.ProcResult; import org.apache.doris.persist.AlterDatabasePropertyInfo; +import org.apache.doris.persist.AlterViewInfo; import org.apache.doris.persist.BarrierLog; import org.apache.doris.persist.BatchModifyPartitionsInfo; import org.apache.doris.persist.BinlogGcInfo; @@ -322,6 +323,43 @@ public void addTruncateTable(TruncateTableInfo info, long commitSeq) { addBinlog(dbId, tableIds, commitSeq, timestamp, type, data, false, record); } + public void addTableRename(TableInfo info, long commitSeq) { + long dbId = info.getDbId(); + long tableId = info.getTableId(); + TBinlogType type = TBinlogType.RENAME_TABLE; + String data = info.toJson(); + BarrierLog log = new BarrierLog(dbId, tableId, type, data); + addBarrierLog(log, commitSeq); + } + + public void addColumnRename(TableRenameColumnInfo info, long commitSeq) { + long dbId = info.getDbId(); + long tableId = info.getTableId(); + TBinlogType type = TBinlogType.RENAME_COLUMN; + String data = info.toJson(); + BarrierLog log = new BarrierLog(dbId, tableId, type, data); + addBarrierLog(log, commitSeq); + } + + public void addModifyComment(ModifyCommentOperationLog info, long commitSeq) { + long dbId = info.getDbId(); + long tableId = info.getTblId(); + TBinlogType type = TBinlogType.MODIFY_COMMENT; + String data = info.toJson(); + BarrierLog log = new BarrierLog(dbId, tableId, type, data); + addBarrierLog(log, commitSeq); + } + + // add Modify view + public void addModifyViewDef(AlterViewInfo alterViewInfo, long commitSeq) { + long dbId = alterViewInfo.getDbId(); + long tableId = alterViewInfo.getTableId(); + TBinlogType type = TBinlogType.MODIFY_VIEW_DEF; + String data = alterViewInfo.toJson(); + BarrierLog log = new BarrierLog(dbId, tableId, type, data); + addBarrierLog(log, commitSeq); + } + // get binlog by dbId, return first binlog.version > version public Pair getBinlog(long dbId, long tableId, long prevCommitSeq) { TStatus status = new TStatus(TStatusCode.OK); @@ -358,33 +396,6 @@ public Pair getBinlogLag(long dbId, long tableId, long prevCommit } } - public void addTableRename(TableInfo info, long commitSeq) { - long dbId = info.getDbId(); - long tableId = info.getTableId(); - TBinlogType type = TBinlogType.RENAME_TABLE; - String data = info.toJson(); - BarrierLog log = new BarrierLog(dbId, tableId, type, data); - addBarrierLog(log, commitSeq); - } - - public void addColumnRename(TableRenameColumnInfo info, long commitSeq) { - long dbId = info.getDbId(); - long tableId = info.getTableId(); - TBinlogType type = TBinlogType.RENAME_COLUMN; - String data = info.toJson(); - BarrierLog log = new BarrierLog(dbId, tableId, type, data); - addBarrierLog(log, commitSeq); - } - - public void addModifyComment(ModifyCommentOperationLog info, long commitSeq) { - long dbId = info.getDbId(); - long tableId = info.getTblId(); - TBinlogType type = TBinlogType.MODIFY_COMMENT; - String data = info.toJson(); - BarrierLog log = new BarrierLog(dbId, tableId, type, data); - addBarrierLog(log, commitSeq); - } - // get the dropped partitions of the db. public List getDroppedPartitions(long dbId) { lock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java index df1fb5696c6b15..c54dfcf7c3100f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/AlterViewInfo.java @@ -105,4 +105,8 @@ public static AlterViewInfo read(DataInput in) throws IOException { String json = Text.readString(in); return GsonUtils.GSON.fromJson(json, AlterViewInfo.class); } + + public String toJson() { + return GsonUtils.GSON.toJson(this); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 54c12dfd0463e5..4ac9028d985bb2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -302,6 +302,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { case OperationType.OP_MODIFY_VIEW_DEF: { AlterViewInfo info = (AlterViewInfo) journal.getData(); env.getAlterInstance().replayModifyViewDef(info); + env.getBinlogManager().addModifyViewDef(info, logId); break; } case OperationType.OP_RENAME_PARTITION: { @@ -1444,7 +1445,11 @@ public void logTableRename(TableInfo tableInfo) { } public void logModifyViewDef(AlterViewInfo alterViewInfo) { - logEdit(OperationType.OP_MODIFY_VIEW_DEF, alterViewInfo); + long logId = logEdit(OperationType.OP_MODIFY_VIEW_DEF, alterViewInfo); + if (LOG.isDebugEnabled()) { + LOG.debug("log modify view, logId : {}, infos: {}", logId, alterViewInfo); + } + Env.getCurrentEnv().getBinlogManager().addModifyViewDef(alterViewInfo, logId); } public void logRollupRename(TableInfo tableInfo) { diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index b58f8a42e8924d..3636a9acf7f1ef 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1030,6 +1030,7 @@ enum TBinlogType { RENAME_TABLE = 14, RENAME_COLUMN = 15, MODIFY_COMMENT = 16, + MODIFY_VIEW_DEF = 17, // Keep some IDs for allocation so that when new binlog types are added in the // future, the changes can be picked back to the old versions without breaking @@ -1046,8 +1047,7 @@ enum TBinlogType { // MODIFY_XXX = 17, // MIN_UNKNOWN = 18, // UNKNOWN_3 = 19, - MIN_UNKNOWN = 17, - UNKNOWN_2 = 18, + MIN_UNKNOWN = 18, UNKNOWN_3 = 19, UNKNOWN_4 = 20, UNKNOWN_5 = 21, From b061b3ef0a78d005c955873610b2abc6eb9e9cfd Mon Sep 17 00:00:00 2001 From: zhangdong Date: Mon, 25 Nov 2024 22:03:49 +0800 Subject: [PATCH 21/31] [fix](auth) after fe restarting, external permissions are lost (#43275) (#43339) pick: https://github.com/apache/doris/pull/43275 --- .../src/main/java/org/apache/doris/mysql/privilege/Auth.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java index 2d2a84c3be2dae..cf856de3b7e2f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java @@ -582,7 +582,9 @@ private void grantInternal(UserIdentity userIdent, String role, TablePattern tbl throws DdlException { writeLock(); try { - checkTablePatternExist(tblPattern); + if (!isReplay) { + checkTablePatternExist(tblPattern); + } if (role == null) { if (!doesUserExist(userIdent)) { throw new DdlException("user " + userIdent + " does not exist"); From 12fb61cb2c6c5f7947bd9c5753e71c6e16ea47a2 Mon Sep 17 00:00:00 2001 From: qiye Date: Tue, 26 Nov 2024 09:55:57 +0800 Subject: [PATCH 22/31] [fix](bloom filter)Fix drop column with bloom filter index (#44361) (#44480) pick #44361 --- .../doris/alter/SchemaChangeHandler.java | 24 +++++- .../test_bloom_filter_drop_column.out | 2 +- .../test_bloom_filter_drop_column.groovy | 74 +++++++++++++++++-- 3 files changed, 92 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 3df21f88be1ea5..eb09d8eb5830c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -414,9 +414,12 @@ private boolean processDropColumn(DropColumnClause alterClause, OlapTable olapTa // drop bloom filter column Set bfCols = olapTable.getCopiedBfColumns(); if (bfCols != null) { - Set newBfCols = new HashSet<>(); + Set newBfCols = null; for (String bfCol : bfCols) { if (!bfCol.equalsIgnoreCase(dropColName)) { + if (newBfCols == null) { + newBfCols = Sets.newHashSet(); + } newBfCols.add(bfCol); } } @@ -2736,6 +2739,25 @@ public void modifyTableLightSchemaChange(String rawSql, Database db, OlapTable o LOG.info("finished modify table's add or drop or modify columns. table: {}, job: {}, is replay: {}", olapTable.getName(), jobId, isReplay); } + // for bloom filter, rebuild bloom filter info by table schema in replay + if (isReplay) { + Set bfCols = olapTable.getCopiedBfColumns(); + if (bfCols != null) { + List columns = olapTable.getBaseSchema(); + Set newBfCols = null; + for (String bfCol : bfCols) { + for (Column column : columns) { + if (column.getName().equalsIgnoreCase(bfCol)) { + if (newBfCols == null) { + newBfCols = Sets.newHashSet(); + } + newBfCols.add(column.getName()); + } + } + } + olapTable.setBloomFilterInfo(newBfCols, olapTable.getBfFpp()); + } + } } public void replayModifyTableLightSchemaChange(TableAddOrDropColumnsInfo info) throws MetaNotFoundException { diff --git a/regression-test/data/bloom_filter_p0/test_bloom_filter_drop_column.out b/regression-test/data/bloom_filter_p0/test_bloom_filter_drop_column.out index 2c6ca8d224b728..14334dfb4b5c48 100644 --- a/regression-test/data/bloom_filter_p0/test_bloom_filter_drop_column.out +++ b/regression-test/data/bloom_filter_p0/test_bloom_filter_drop_column.out @@ -1,6 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !select -- -1 1 +1 1 1 -- !select -- 1 \N diff --git a/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy b/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy index a18ff9fdbe0d80..d83c70af30c709 100644 --- a/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy +++ b/regression-test/suites/bloom_filter_p0/test_bloom_filter_drop_column.groovy @@ -21,31 +21,93 @@ suite("test_bloom_filter_drop_column") { sql """CREATE TABLE IF NOT EXISTS ${table_name} ( `a` varchar(150) NULL, - `c1` varchar(10) + `c1` varchar(10), + `c2` varchar(10) ) ENGINE=OLAP DUPLICATE KEY(`a`) DISTRIBUTED BY HASH(`a`) BUCKETS 1 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", - "bloom_filter_columns" = "c1", + "bloom_filter_columns" = "c1, c2", "in_memory" = "false", "storage_format" = "V2" )""" + def timeout = 60000 + def delta_time = 1000 + def alter_res = "null" + def useTime = 0 - sql """INSERT INTO ${table_name} values ('1', '1')""" + def wait_for_latest_op_on_table_finish = { tableName, OpTimeout -> + for(int t = delta_time; t <= OpTimeout; t += delta_time){ + alter_res = sql """SHOW ALTER TABLE COLUMN WHERE TableName = "${tableName}" ORDER BY CreateTime DESC LIMIT 1;""" + alter_res = alter_res.toString() + if(alter_res.contains("FINISHED")) { + sleep(3000) // wait change table state to normal + logger.info(table_name + " latest alter job finished, detail: " + alter_res) + break + } + useTime = t + sleep(delta_time) + } + assertTrue(useTime <= OpTimeout, "wait_for_latest_op_on_table_finish timeout") + } + + def assertShowCreateTableWithRetry = { tableName, expectedCondition, contains, maxRetries, waitSeconds -> + int attempt = 0 + while (attempt < maxRetries) { + def res = sql """SHOW CREATE TABLE ${tableName}""" + log.info("Attempt ${attempt + 1}: show table: ${res}") + if (res && res.size() > 0 && ((contains && res[0][1].contains(expectedCondition)) || (!contains && !res[0][1].contains(expectedCondition)))) { + logger.info("Attempt ${attempt + 1}: Condition met.") + return + } else { + logger.warn("Attempt ${attempt + 1}: Condition not met. Retrying after ${waitSeconds} second(s)...") + } + attempt++ + if (attempt < maxRetries) { + sleep(waitSeconds * 1000) + } + } + def finalRes = sql """SHOW CREATE TABLE ${tableName}""" + log.info("Final attempt: show table: ${finalRes}") + assertTrue(finalRes && finalRes.size() > 0, "SHOW CREATE TABLE return empty or null") + if (contains) { + assertTrue(finalRes[0][1].contains(expectedCondition), "expected to contain \"${expectedCondition}\", actual: ${finalRes[0][1]}") + } else { + assertTrue(!finalRes[0][1].contains(expectedCondition), "expected not to contain \"${expectedCondition}\", actual: ${finalRes[0][1]}") + } + } + + sql """INSERT INTO ${table_name} values ('1', '1', '1')""" + sql "sync" qt_select """select * from ${table_name} order by a""" + assertShowCreateTableWithRetry(table_name, "\"bloom_filter_columns\" = \"c1, c2\"", true, 3, 30) // drop column c1 sql """ALTER TABLE ${table_name} DROP COLUMN c1""" - // show create table - def res = sql """SHOW CREATE TABLE ${table_name}""" - assert res[0][1].contains("\"bloom_filter_columns\" = \"\"") + wait_for_latest_op_on_table_finish(table_name, timeout) + sql "sync" + + // show create table with retry logic + assertShowCreateTableWithRetry(table_name, "\"bloom_filter_columns\" = \"c2\"", true, 3, 30) + + // drop column c2 + sql """ALTER TABLE ${table_name} DROP COLUMN c2""" + wait_for_latest_op_on_table_finish(table_name, timeout) + sql "sync" + + // show create table with retry logic + assertShowCreateTableWithRetry(table_name, "\"bloom_filter_columns\" = \"\"", false, 3, 30) // add new column c1 sql """ALTER TABLE ${table_name} ADD COLUMN c1 ARRAY""" + wait_for_latest_op_on_table_finish(table_name, timeout) + sql "sync" + // insert data sql """INSERT INTO ${table_name} values ('2', null)""" + sql "sync" // select data qt_select """select * from ${table_name} order by a""" } From 90da61b3e763ffbc4d1981e81c0df048e1f3a62e Mon Sep 17 00:00:00 2001 From: Lijia Liu Date: Wed, 27 Nov 2024 11:39:08 +0800 Subject: [PATCH 23/31] [fix](nereids)change lag/lead function signature (#40060) (#44611) pick from master #40060 remove CharType from the signature Co-authored-by: starocean999 <40539150+starocean999@users.noreply.github.com> --- .../functions/window/RequireTrivialTypes.java | 2 -- .../lag_lead_signature.groovy | 35 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 regression-test/suites/nereids_syntax_p0/lag_lead_signature.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java index 34295732d58263..04d92fdeb11132 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/window/RequireTrivialTypes.java @@ -19,7 +19,6 @@ import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.BooleanType; -import org.apache.doris.nereids.types.CharType; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -64,7 +63,6 @@ public interface RequireTrivialTypes { DateTimeV2Type.SYSTEM_DEFAULT, TimeType.INSTANCE, TimeV2Type.INSTANCE, - CharType.SYSTEM_DEFAULT, VarcharType.SYSTEM_DEFAULT, StringType.INSTANCE ); diff --git a/regression-test/suites/nereids_syntax_p0/lag_lead_signature.groovy b/regression-test/suites/nereids_syntax_p0/lag_lead_signature.groovy new file mode 100644 index 00000000000000..5dde98a95a0f5c --- /dev/null +++ b/regression-test/suites/nereids_syntax_p0/lag_lead_signature.groovy @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("lag_lead_signature") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + sql """drop table if exists lag_lead_signature_t""" + sql """ CREATE TABLE lag_lead_signature_t ( + `k` VARCHAR(200) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(`k`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + );""" + + sql "insert into lag_lead_signature_t values ('44'), ('28');" + + sql "select lag(k, 1, 0) over(), lead(k, 1, 0) over() from lag_lead_signature_t;" + +} \ No newline at end of file From 9cef7ec0c573f483cc67fffa4bc88c9d3effd918 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 27 Nov 2024 18:35:49 +0800 Subject: [PATCH 24/31] [fix](case)Fix multiple case tables with the same name causing case failure (#42203) (#43043) pick from master #42203 --- .../suites/nereids_p0/union/test_union.groovy | 12 +++++++----- .../suites/query_p0/union/test_union.groovy | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/regression-test/suites/nereids_p0/union/test_union.groovy b/regression-test/suites/nereids_p0/union/test_union.groovy index fda90bc1cab30b..259bad406d731d 100644 --- a/regression-test/suites/nereids_p0/union/test_union.groovy +++ b/regression-test/suites/nereids_p0/union/test_union.groovy @@ -18,6 +18,8 @@ suite("test_union") { sql "SET enable_nereids_planner=true" sql "SET enable_fallback_to_original_planner=false" + String suiteName = "nereids_union_test_union" + String viewName = "${suiteName}_view" def db = "nereids_test_query_db" sql "use ${db}" @@ -174,14 +176,14 @@ suite("test_union") { // test_union_bug // PALO-3617 qt_union36 """select * from (select 1 as a, 2 as b union select 3, 3) c where a = 1""" - sql """drop view if exists nullable""" - sql """CREATE VIEW `nullable` AS SELECT `a`.`k1` AS `n1`, `b`.`k2` AS `n2` + sql """drop view if exists ${viewName}""" + sql """CREATE VIEW `${viewName}` AS SELECT `a`.`k1` AS `n1`, `b`.`k2` AS `n2` FROM `default_cluster:${db}`.`baseall` a LEFT OUTER JOIN `default_cluster:${db}`.`bigtable` b ON `a`.`k1` = `b`.`k1` + 10 WHERE `b`.`k2` IS NULL""" - order_qt_union37 """select n1 from nullable union all select n2 from nullable""" - qt_union38 """(select n1 from nullable) union all (select n2 from nullable order by n1) order by n1""" - qt_union39 """(select n1 from nullable) union all (select n2 from nullable) order by n1""" + order_qt_union37 """select n1 from ${viewName} union all select n2 from ${viewName}""" + qt_union38 """(select n1 from ${viewName}) union all (select n2 from ${viewName} order by n1) order by n1""" + qt_union39 """(select n1 from ${viewName}) union all (select n2 from ${viewName}) order by n1""" // test_union_different_column diff --git a/regression-test/suites/query_p0/union/test_union.groovy b/regression-test/suites/query_p0/union/test_union.groovy index d67e3aa65c309a..fc0959bf943aee 100644 --- a/regression-test/suites/query_p0/union/test_union.groovy +++ b/regression-test/suites/query_p0/union/test_union.groovy @@ -16,6 +16,8 @@ // under the License. suite("test_union") { + String suiteName = "query_union_test_union" + String viewName = "${suiteName}_view" def db = "test_query_db" sql "use ${db}" @@ -172,14 +174,14 @@ suite("test_union") { // test_union_bug // PALO-3617 qt_union36 """select * from (select 1 as a, 2 as b union select 3, 3) c where a = 1""" - sql """drop view if exists nullable""" - sql """CREATE VIEW `nullable` AS SELECT `a`.`k1` AS `n1`, `b`.`k2` AS `n2` + sql """drop view if exists ${viewName}""" + sql """CREATE VIEW `${viewName}` AS SELECT `a`.`k1` AS `n1`, `b`.`k2` AS `n2` FROM `default_cluster:${db}`.`baseall` a LEFT OUTER JOIN `default_cluster:${db}`.`bigtable` b ON `a`.`k1` = `b`.`k1` + 10 WHERE `b`.`k2` IS NULL""" - order_qt_union37 """select n1 from nullable union all select n2 from nullable""" - qt_union38 """(select n1 from nullable) union all (select n2 from nullable order by n1) order by n1""" - qt_union39 """(select n1 from nullable) union all (select n2 from nullable) order by n1""" + order_qt_union37 """select n1 from ${viewName} union all select n2 from ${viewName}""" + qt_union38 """(select n1 from ${viewName}) union all (select n2 from ${viewName} order by n1) order by n1""" + qt_union39 """(select n1 from ${viewName}) union all (select n2 from ${viewName}) order by n1""" // test_union_different_column From 5a0b5839727ba8c9504e84cd759dc0e45b292522 Mon Sep 17 00:00:00 2001 From: zhangdong Date: Wed, 27 Nov 2024 20:07:35 +0800 Subject: [PATCH 25/31] [fix](insertoverwrite)fix roll up will lose after insert overwrite (#41888) The implementation method on NEREIDS is different, so there is no problem, so it was only fixed in 2.0 --- .../org/apache/doris/qe/StmtExecutor.java | 2 +- .../insert_overwrite_rollup.groovy | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/insert_overwrite_p0/insert_overwrite_rollup.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 0e09640ba2c8f3..89ca99fb8dc5f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -2581,7 +2581,7 @@ private void handleOverwriteTable(InsertOverwriteTableStmt iotStmt) { TableName targetTableName = new TableName(null, iotStmt.getDb(), iotStmt.getTbl()); try { // create a tmp table with uuid - parsedStmt = new CreateTableLikeStmt(false, tmpTableName, targetTableName, null, false); + parsedStmt = new CreateTableLikeStmt(false, tmpTableName, targetTableName, null, true); parsedStmt.setUserInfo(context.getCurrentUserIdentity()); execute(); // if create tmp table err, return diff --git a/regression-test/suites/insert_overwrite_p0/insert_overwrite_rollup.groovy b/regression-test/suites/insert_overwrite_p0/insert_overwrite_rollup.groovy new file mode 100644 index 00000000000000..64a24f28e98ef3 --- /dev/null +++ b/regression-test/suites/insert_overwrite_p0/insert_overwrite_rollup.groovy @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("insert_overwrite_rollup","p0") { + String suiteName = "insert_overwrite_not_fallback" + String tableName = "${suiteName}_table" + String rollupName = "${suiteName}_rollup" + sql """drop table if exists `${tableName}`""" + sql """ + CREATE TABLE `${tableName}` + ( + k2 TINYINT, + k3 INT not null + ) + COMMENT "my first table" + PARTITION BY LIST(`k3`) + ( + PARTITION `p1` VALUES IN ('1'), + PARTITION `p2` VALUES IN ('2'), + PARTITION `p3` VALUES IN ('3') + ) + DISTRIBUTED BY HASH(k2) BUCKETS 2 + ROLLUP ( + ${rollupName}(k3) + ) + PROPERTIES ( + "replication_num" = "1" + ); + """ + sql """ + insert overwrite table ${tableName} values(1,1); + """ + def res = sql "desc ${tableName} all" + logger.info("res: " + res.toString()) + assertTrue(res.toString().contains("${rollupName}")) + sql """drop table if exists `${tableName}`""" +} + From 546c7677652859a0f711d9771973fca71026cb0a Mon Sep 17 00:00:00 2001 From: zhannngchen Date: Wed, 27 Nov 2024 22:11:12 +0800 Subject: [PATCH 26/31] [fix](unique key) getEnableUniqueKeyMergeOnWrite should check key type (#44688) --- .../src/main/java/org/apache/doris/catalog/OlapTable.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 9b95b1b20ec6ec..ffb9a0605afa5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -2418,6 +2418,9 @@ public boolean getEnableUniqueKeyMergeOnWrite() { if (tableProperty == null) { return false; } + if (getKeysType() != KeysType.UNIQUE_KEYS) { + return false; + } return tableProperty.getEnableUniqueKeyMergeOnWrite(); } From 4f0288f2303c142058726454a4ec5fc8dd47623e Mon Sep 17 00:00:00 2001 From: starocean999 Date: Thu, 28 Nov 2024 14:27:38 +0800 Subject: [PATCH 27/31] [fix](nereids)the column name should be case insensitive in tablet prune (#44207) pick from master https://github.com/apache/doris/pull/44064 ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../rules/rewrite/PruneOlapScanTablet.java | 4 +-- .../org/apache/doris/planner/ScanNode.java | 3 +- .../planner/HashDistributionPrunerTest.java | 24 +++++++------- .../doris/planner/OlapScanNodeTest.java | 5 +-- .../tablet_prune/test_tablet_prune.groovy | 32 +++++++++++++++++++ 5 files changed, 51 insertions(+), 17 deletions(-) create mode 100644 regression-test/suites/nereids_rules_p0/tablet_prune/test_tablet_prune.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java index 0b079d781687ec..7e82d5d0be7bde 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanTablet.java @@ -34,7 +34,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; -import com.google.common.collect.Maps; +import org.apache.commons.collections.map.CaseInsensitiveMap; import java.util.Collection; import java.util.HashSet; @@ -82,7 +82,7 @@ private Collection getSelectedTabletIds(Set expressions, return index.getTabletIdsInOrder(); } HashDistributionInfo hashInfo = (HashDistributionInfo) info; - Map filterMap = Maps.newHashMap(); + Map filterMap = new CaseInsensitiveMap(); expressions.stream().map(ExpressionUtils::checkAndMaybeCommute).filter(Optional::isPresent) .forEach(expr -> new ExpressionColumnFilterConverter(filterMap).convert(expr.get())); return new HashDistributionPruner(index.getTabletIdsInOrder(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index 2f309d9e498746..b6e7c80d4dc547 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -61,6 +61,7 @@ import com.google.common.collect.RangeSet; import com.google.common.collect.Sets; import com.google.common.collect.TreeRangeSet; +import org.apache.commons.collections.map.CaseInsensitiveMap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -78,7 +79,7 @@ public abstract class ScanNode extends PlanNode { private static final Logger LOG = LogManager.getLogger(ScanNode.class); protected final TupleDescriptor desc; // for distribution prunner - protected Map columnFilters = Maps.newHashMap(); + protected Map columnFilters = new CaseInsensitiveMap(); // Use this if partition_prune_algorithm_version is 2. protected Map columnNameToRange = Maps.newHashMap(); protected String sortColumn = null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java index 2da47314d6871a..e2896ee239091e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/HashDistributionPrunerTest.java @@ -26,8 +26,8 @@ import org.apache.doris.catalog.PrimitiveType; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.commons.collections.map.CaseInsensitiveMap; import org.junit.Assert; import org.junit.Test; @@ -84,12 +84,12 @@ public void test() { inList4.add(new StringLiteral("2")); shopTypeFilter.setInPredicate(new InPredicate(new SlotRef(null, "shop_type"), inList4, false)); - Map filters = Maps.newHashMap(); - filters.put("dealDate", dealDatefilter); - filters.put("main_brand_id", mainBrandFilter); - filters.put("item_third_cate_id", itemThirdFilter); - filters.put("channel", channelFilter); - filters.put("shop_type", shopTypeFilter); + Map filters = new CaseInsensitiveMap(); + filters.put("DEALDATE", dealDatefilter); + filters.put("MAIN_BRAND_ID", mainBrandFilter); + filters.put("ITEM_THIRD_CATE_ID", itemThirdFilter); + filters.put("CHANNEL", channelFilter); + filters.put("SHOP_TYPE", shopTypeFilter); HashDistributionPruner pruner = new HashDistributionPruner(tabletIds, columns, filters, tabletIds.size(), true); @@ -97,16 +97,16 @@ public void test() { // 20 = 1 * 5 * 2 * 2 * 1 (element num of each filter) Assert.assertEquals(20, results.size()); - filters.get("shop_type").getInPredicate().addChild(new StringLiteral("4")); + filters.get("SHOP_TYPE").getInPredicate().addChild(new StringLiteral("4")); results = pruner.prune(); // 40 = 1 * 5 * 2 * 2 * 2 (element num of each filter) // 39 is because these is hash conflict Assert.assertEquals(39, results.size()); - filters.get("shop_type").getInPredicate().addChild(new StringLiteral("5")); - filters.get("shop_type").getInPredicate().addChild(new StringLiteral("6")); - filters.get("shop_type").getInPredicate().addChild(new StringLiteral("7")); - filters.get("shop_type").getInPredicate().addChild(new StringLiteral("8")); + filters.get("SHOP_TYPE").getInPredicate().addChild(new StringLiteral("5")); + filters.get("SHOP_TYPE").getInPredicate().addChild(new StringLiteral("6")); + filters.get("SHOP_TYPE").getInPredicate().addChild(new StringLiteral("7")); + filters.get("SHOP_TYPE").getInPredicate().addChild(new StringLiteral("8")); results = pruner.prune(); // 120 = 1 * 5 * 2 * 2 * 6 (element num of each filter) > 100 Assert.assertEquals(300, results.size()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java index d8d785435585f8..417cf8b55d02fe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java @@ -31,6 +31,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.commons.collections.map.CaseInsensitiveMap; import org.junit.Assert; import org.junit.Test; @@ -62,8 +63,8 @@ public void testHashDistributionOneUser() throws AnalysisException { PartitionColumnFilter columnFilter = new PartitionColumnFilter(); columnFilter.setInPredicate(inPredicate); - Map filterMap = Maps.newHashMap(); - filterMap.put("columnA", columnFilter); + Map filterMap = new CaseInsensitiveMap(); + filterMap.put("COLUMNA", columnFilter); DistributionPruner partitionPruner = new HashDistributionPruner( partitions, diff --git a/regression-test/suites/nereids_rules_p0/tablet_prune/test_tablet_prune.groovy b/regression-test/suites/nereids_rules_p0/tablet_prune/test_tablet_prune.groovy new file mode 100644 index 00000000000000..6b69d86bf1f141 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/tablet_prune/test_tablet_prune.groovy @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_tablet_prune") { + sql "SET enable_nereids_planner=true" + sql "SET enable_fallback_to_original_planner=false" + + sql "drop table if exists t_customers_wide_index" + sql """ + CREATE TABLE `t_customers_wide_index` ( `CUSTOMER_ID` int NULL, `ADDRESS` varchar(1500) NULL) ENGINE=OLAP UNIQUE KEY(`CUSTOMER_ID`) DISTRIBUTED BY HASH(`CUSTOMER_ID`) BUCKETS 32 PROPERTIES ( "replication_allocation" = "tag.location.default: 1", "file_cache_ttl_seconds" = "0", "is_being_synced" = "false", "storage_medium" = "hdd", "storage_format" = "V2", "inverted_index_storage_format" = "V2", "enable_unique_key_merge_on_write" = "true", "light_schema_change" = "true", "store_row_column" = "true", "row_store_page_size" = "16384", "disable_auto_compaction" = "false", "enable_single_replica_compaction" = "false", "group_commit_interval_ms" = "10000", "group_commit_data_bytes" = "134217728", "enable_mow_light_delete" = "false" ); """ + sql """ + insert into t_customers_wide_index values (1, "111"); + """ + explain { + sql("SELECT * from t_customers_wide_index WHERE customer_id = 1817422;") + contains "tablets=1/32" + } +} \ No newline at end of file From db2c4d4738ed1c673b198bf659a02b27055158b8 Mon Sep 17 00:00:00 2001 From: camby Date: Thu, 5 Dec 2024 18:48:36 +0800 Subject: [PATCH 28/31] [fix](api) make QueryDetailQueue.queryCapacity configurable (#44990) ### What problem does this PR solve? Problem: very large SQLs make FE OOM, here make the capacity configurable. In branch-2.1 we already removed the usage by pr: https://github.com/apache/doris/pull/29999 --- .../src/main/java/org/apache/doris/common/Config.java | 6 ++++++ .../main/java/org/apache/doris/qe/QueryDetailQueue.java | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index dd09db27d94ba1..94c6ad4b0aa6b9 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2464,6 +2464,12 @@ public class Config extends ConfigBase { }) public static int http_load_submitter_max_worker_threads = 2; + @ConfField(mutable = false, masterOnly = false, description = { + "缓存的最大Query数量，用于响应http请求/api/query_detail。", + "The max capacity of queries for query_detail api." + }) + public static int http_query_detail_capacity = 10000; + @ConfField(mutable = true, masterOnly = true, description = { "load label个数阈值，超过该个数后，对于已经完成导入作业或者任务，" + "其label会被删除，被删除的 label 可以被重用。 值为 -1 时，表示此阈值不生效。", diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/QueryDetailQueue.java b/fe/fe-core/src/main/java/org/apache/doris/qe/QueryDetailQueue.java index 85f09798a33116..b0d464df337478 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/QueryDetailQueue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/QueryDetailQueue.java @@ -17,6 +17,8 @@ package org.apache.doris.qe; +import org.apache.doris.common.Config; + import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -31,9 +33,12 @@ public class QueryDetailQueue { private static Map runningQueries = Maps.newHashMap(); private static LinkedList totalQueries = new LinkedList(); - private static int queryCapacity = 10000; + private static int queryCapacity = Config.http_query_detail_capacity; public static synchronized void addOrUpdateQueryDetail(QueryDetail queryDetail) { + if (queryCapacity <= 0) { + return; + } if (runningQueries.get(queryDetail.getQueryId()) == null) { if (queryDetail.getState() == QueryDetail.QueryMemState.RUNNING) { runningQueries.put(queryDetail.getQueryId(), queryDetail); From b9bc7925779e34e3e969a28f694236b4af971f01 Mon Sep 17 00:00:00 2001 From: Pxl Date: Fri, 13 Dec 2024 13:04:02 +0800 Subject: [PATCH 29/31] =?UTF-8?q?[Bug](runtime-filter)=20set=20inited=20to?= =?UTF-8?q?=20true=20on=20BloomFilterFuncBase::assi=E2=80=A6=20=20=20#3933?= =?UTF-8?q?5=20(#45388)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …gn (#39335) ## Proposed changes set inited to true on BloomFilterFuncBase::assign ```cpp *** SIGABRT unknown detail explain (@0x1ba2) received by PID 7074 (TID 9018 OR 0x7f4adf5df640) from PID 7074; stack trace: *** 0# doris::signal::(anonymous namespace)::FailureSignalHandler(int, siginfo_t*, void*) at /home/zcp/repo_center/doris_master/doris/be/src/common/signal_handler.h:421 1# 0x00007F525A7F6520 in /lib/x86_64-linux-gnu/libc.so.6 2# pthread_kill at ./nptl/pthread_kill.c:89 3# raise at ../sysdeps/posix/raise.c:27 4# abort at ./stdlib/abort.c:81 5# 0x0000560BDC6CE8DD in /mnt/hdd01/ci/master-deploy/be/lib/doris_be 6# 0x0000560BDC6C0F1A in /mnt/hdd01/ci/master-deploy/be/lib/doris_be 7# google::LogMessage::SendToLog() in /mnt/hdd01/ci/master-deploy/be/lib/doris_be 8# google::LogMessage::Flush() in /mnt/hdd01/ci/master-deploy/be/lib/doris_be 9# google::LogMessageFatal::~LogMessageFatal() in /mnt/hdd01/ci/master-deploy/be/lib/doris_be 10# doris::BloomFilterFuncBase::merge(doris::BloomFilterFuncBase*) at /home/zcp/repo_center/doris_master/doris/be/src/exprs/bloom_filter_func.h:159 11# doris::RuntimePredicateWrapper::merge(doris::RuntimePredicateWrapper const*) at /home/zcp/repo_center/doris_master/doris/be/src/exprs/runtime_filter.cpp:539 12# doris::IRuntimeFilter::merge_from(doris::RuntimePredicateWrapper const*) in /mnt/hdd01/ci/master-deploy/be/lib/doris_be 13# doris::RuntimeFilterMergeControllerEntity::merge(doris::PMergeFilterRequest const*, butil::IOBufAsZeroCopyInputStream*) at /home/zcp/repo_center/doris_master/doris/be/src/runtime/runtime_filter_mgr.cpp:399 14# doris::FragmentMgr::merge_filter(doris::PMergeFilterRequest const*, butil::IOBufAsZeroCopyInputStream*) at /home/zcp/repo_center/doris_master/doris/be/src/runtime/fragment_mgr.cpp:1170 15# std::_Function_handler::_M_invoke(std::_Any_data const&) at /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291 16# doris::WorkThreadPool::work_thread(int) at /home/zcp/repo_center/doris_master/doris/be/src/util/work_thread_pool.hpp:159 17# execute_native_thread_routine at ../../../../../libstdc++-v3/src/c++11/thread.cc:84 18# start_thread at ./nptl/pthread_create.c:442 19# 0x00007F525A8DA850 at ../sysdeps/unix/sysv/linux/x86_64/clone3.S:83 172.20.50.47 last coredump sql: ``` ### What problem does this PR solve? Issue Number: close #xxx Related PR: #xxx Problem Summary: ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [ ] No. - [ ] Yes. - Does this need documentation? - [ ] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- be/src/exprs/bloom_filter_func.h | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h index 7713d18715287d..6810c265947d0e 100644 --- a/be/src/exprs/bloom_filter_func.h +++ b/be/src/exprs/bloom_filter_func.h @@ -196,6 +196,7 @@ class BloomFilterFuncBase : public FilterFuncBase { } _bloom_filter_alloced = data_size; + _inited = true; return _bloom_filter->init(data, data_size); } From 28f7573f589ca945b356665337961977d529bb5f Mon Sep 17 00:00:00 2001 From: Sun Chenyang Date: Tue, 17 Dec 2024 10:57:45 +0800 Subject: [PATCH 30/31] [fix](inverted index) fix rowset data size when enable index compaction (#45350) ### What problem does this PR solve? Problem Summary: - After performing index compaction, the index file size is not included in the rowset's data size. - A similar PR #37232 has already fixed this bug in the master branch. --- be/src/olap/compaction.cpp | 59 ++++++- .../test_index_compaction_rowset_size.groovy | 144 ++++++++++++++++++ 2 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 regression-test/suites/fault_injection_p0/test_index_compaction_rowset_size.groovy diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp index d7f28436800fef..b4ace1f5170595 100644 --- a/be/src/olap/compaction.cpp +++ b/be/src/olap/compaction.cpp @@ -524,6 +524,9 @@ Status Compaction::do_compaction_impl(int64_t permits) { auto& fs = _output_rowset->rowset_meta()->fs(); auto& tablet_path = _tablet->tablet_path(); + // After doing index compaction, need to add this size to rowset->total_size + int64_t compacted_index_file_size = 0; + // we choose the first destination segment name as the temporary index writer path // Used to distinguish between different index compaction auto index_writer_path = tablet_path + "/" + dest_index_files[0]; @@ -536,7 +539,7 @@ Status Compaction::do_compaction_impl(int64_t permits) { ctx.skip_inverted_index.cbegin(), ctx.skip_inverted_index.cend(), [&src_segment_num, &dest_segment_num, &index_writer_path, &src_index_files, &dest_index_files, &fs, &tablet_path, &trans_vec, &dest_segment_num_rows, - &status, this](int32_t column_uniq_id) { + &status, &compacted_index_file_size, this](int32_t column_uniq_id) { auto error_handler = [this](int64_t index_id, int64_t column_uniq_id) { LOG(WARNING) << "failed to do index compaction" << ". tablet=" << _tablet->tablet_id() @@ -584,6 +587,25 @@ Status Compaction::do_compaction_impl(int64_t permits) { error_handler(index_id, column_uniq_id); status = Status::Error( st.msg()); + } else { + for (int i = 0; i < dest_segment_num; ++i) { + // format: rowsetId_segmentId_columnId + auto seg_path = + std::static_pointer_cast(_output_rowset) + ->segment_file_path(i); + std::string index_path = + InvertedIndexDescriptor::get_index_file_name(seg_path, + index_id); + int64_t current_size = 0; + st = fs->file_size(index_path, ¤t_size); + if (!st.ok()) { + error_handler(index_id, column_uniq_id); + status = Status::Error< + ErrorCode::INVERTED_INDEX_COMPACTION_ERROR>( + st.msg()); + } + compacted_index_file_size += current_size; + } } } catch (CLuceneError& e) { error_handler(index_id, column_uniq_id); @@ -597,6 +619,41 @@ Status Compaction::do_compaction_impl(int64_t permits) { return status; } + // index compaction should update total disk size and index disk size= + _output_rowset->rowset_meta()->set_data_disk_size( + _output_rowset->rowset_meta()->data_disk_size() + compacted_index_file_size); + _output_rowset->rowset_meta()->set_total_disk_size( + _output_rowset->rowset_meta()->total_disk_size() + compacted_index_file_size); + _output_rowset->rowset_meta()->set_index_disk_size(_output_rowset->index_disk_size() + + compacted_index_file_size); + + DBUG_EXECUTE_IF("check_after_compaction_file_size", { + int64_t total_file_size = 0; + for (int i = 0; i < dest_segment_num; ++i) { + auto seg_path = std::static_pointer_cast(_output_rowset) + ->segment_file_path(i); + int64_t current_size = 0; + RETURN_IF_ERROR(fs->file_size(seg_path, ¤t_size)); + total_file_size += current_size; + for (auto& column : _cur_tablet_schema->columns()) { + const TabletIndex* index_meta = + _cur_tablet_schema->get_inverted_index(column.unique_id()); + if (index_meta) { + std::string index_path = InvertedIndexDescriptor::get_index_file_name( + seg_path, index_meta->index_id()); + RETURN_IF_ERROR(fs->file_size(index_path, ¤t_size)); + total_file_size += current_size; + } + } + } + if (total_file_size != _output_rowset->rowset_meta()->data_disk_size()) { + Status::Error( + "total file size {} is not equal rowset meta size {}", total_file_size, + _output_rowset->rowset_meta()->data_disk_size()); + } + LOG(INFO) << "succeed to check index compaction file size"; + }) + LOG(INFO) << "succeed to do index compaction" << ". tablet=" << _tablet->full_name() << ", input row number=" << _input_row_num diff --git a/regression-test/suites/fault_injection_p0/test_index_compaction_rowset_size.groovy b/regression-test/suites/fault_injection_p0/test_index_compaction_rowset_size.groovy new file mode 100644 index 00000000000000..9dd3b1fa09a7a1 --- /dev/null +++ b/regression-test/suites/fault_injection_p0/test_index_compaction_rowset_size.groovy @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import org.awaitility.Awaitility + +suite("test_index_compaction_rowset_size", "p0, nonConcurrent") { + + def show_table_name = "test_index_compaction_rowset_size" + + def backendId_to_backendIP = [:] + def backendId_to_backendHttpPort = [:] + getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); + def set_be_config = { key, value -> + for (String backend_id: backendId_to_backendIP.keySet()) { + def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) + logger.info("update config: code=" + code + ", out=" + out + ", err=" + err) + } + } + set_be_config.call("inverted_index_compaction_enable", "true") + + sql "DROP TABLE IF EXISTS ${show_table_name}" + sql """ + CREATE TABLE ${show_table_name} ( + `@timestamp` int(11) NULL, + `clientip` varchar(20) NULL, + `request` varchar(500) NULL, + `status` int NULL, + `size` int NULL, + INDEX clientip_idx (`clientip`) USING INVERTED COMMENT '', + INDEX request_idx (`request`) USING INVERTED PROPERTIES("parser" = "unicode") COMMENT '', + INDEX size_idx (`size`) USING INVERTED COMMENT '', + ) ENGINE=OLAP + DUPLICATE KEY(`@timestamp`, `clientip`) + DISTRIBUTED BY HASH(`@timestamp`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "compaction_policy" = "time_series", + "time_series_compaction_file_count_threshold" = "20", + "disable_auto_compaction" = "true" + ); + """ + + def compaction = { + + def tablets = sql_return_maparray """ show tablets from ${show_table_name}; """ + + for (def tablet in tablets) { + int beforeSegmentCount = 0 + String tablet_id = tablet.TabletId + (code, out, err) = curl("GET", tablet.CompactionStatus) + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + beforeSegmentCount += Integer.parseInt(rowset.split(" ")[1]) + } + assertEquals(beforeSegmentCount, 12) + } + + // trigger compactions for all tablets in ${tableName} + for (def tablet in tablets) { + String tablet_id = tablet.TabletId + backend_id = tablet.BackendId + (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactJson = parseJson(out.trim()) + assertEquals("success", compactJson.status.toLowerCase()) + } + + // wait for all compactions done + for (def tablet in tablets) { + Awaitility.await().atMost(10, TimeUnit.MINUTES).untilAsserted(() -> { + Thread.sleep(5000) + String tablet_id = tablet.TabletId + backend_id = tablet.BackendId + (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) + logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def compactionStatus = parseJson(out.trim()) + assertEquals("compaction task for this tablet is not running", compactionStatus.msg.toLowerCase()) + }); + } + + + for (def tablet in tablets) { + int afterSegmentCount = 0 + String tablet_id = tablet.TabletId + (code, out, err) = curl("GET", tablet.CompactionStatus) + logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err) + assertEquals(code, 0) + def tabletJson = parseJson(out.trim()) + assert tabletJson.rowsets instanceof List + for (String rowset in (List) tabletJson.rowsets) { + logger.info("rowset is: " + rowset) + afterSegmentCount += Integer.parseInt(rowset.split(" ")[1]) + } + assertEquals(afterSegmentCount, 1) + } + } + + + + // 1. load data + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + sql """ INSERT INTO ${show_table_name} VALUES (100, "andy", "andy love apple", 100, 200); """ + + try { + GetDebugPoint().enableDebugPointForAllBEs("check_after_compaction_file_size") + // 2. compaction + compaction.call() + } finally { + GetDebugPoint().disableDebugPointForAllBEs("check_after_compaction_file_size") + } + + +} From fdff4a643ad3a0dfab1dba37abbfb7105907703f Mon Sep 17 00:00:00 2001 From: yujun Date: Fri, 20 Dec 2024 10:59:30 +0800 Subject: [PATCH 31/31] branch-2.0: [fix](create table) fix create table fail msg #45623 (#45653) cherry pick from #45623 --- .../apache/doris/clone/TabletScheduler.java | 2 +- .../java/org/apache/doris/system/Backend.java | 3 +++ .../apache/doris/system/SystemInfoService.java | 2 +- .../apache/doris/catalog/CreateTableTest.java | 18 ++++++++++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java index 9dd4b22c436530..43bb5bd4e9b6af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java @@ -1444,7 +1444,7 @@ private RootPathLoadStatistic chooseAvailableDestPath(TabletSchedCtx tabletCtx, !allFitPathsSameMedium.isEmpty() ? allFitPathsSameMedium : allFitPathsDiffMedium; if (allFitPaths.isEmpty()) { List backendsInfo = Env.getCurrentSystemInfo().getAllBackends().stream() - .filter(be -> be.getLocationTag() == tag) + .filter(be -> be.getLocationTag().equals(tag)) .map(Backend::getDetailsForCreateReplica) .collect(Collectors.toList()); throw new SchedException(Status.UNRECOVERABLE, String.format("unable to find dest path for new replica" diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java index ae93b7faab6931..1f51c2d16ff71a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/Backend.java @@ -27,6 +27,7 @@ import org.apache.doris.common.util.PrintableMap; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.SimpleScheduler; import org.apache.doris.resource.Tag; import org.apache.doris.system.HeartbeatResponse.HbStatus; import org.apache.doris.thrift.TDisk; @@ -270,6 +271,8 @@ public String getDetailsForCreateReplica() { sb.append(", isDecommissioned=true, exclude it"); } else if (isComputeNode()) { sb.append(", isComputeNode=true, exclude it"); + } else if (!Config.disable_backend_black_list && !SimpleScheduler.isAvailable(this)) { + sb.append(", is in black list, exclude it"); } else { sb.append(", hdd disks count={"); if (hddOk > 0) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java index 6b4c2bee09f997..87d7fd5ce52065 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java @@ -540,7 +540,7 @@ public String getDetailsForCreateReplica(ReplicaAllocation replicaAlloc) { StringBuilder sb = new StringBuilder(" Backends details: "); for (Tag tag : replicaAlloc.getAllocMap().keySet()) { sb.append("backends with tag ").append(tag).append(" is "); - sb.append(idToBackendRef.values().stream().filter(be -> be.getLocationTag() == tag) + sb.append(idToBackendRef.values().stream().filter(be -> be.getLocationTag().equals(tag)) .map(Backend::getDetailsForCreateReplica) .collect(Collectors.toList())); sb.append(", "); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java index 15745a700712d7..7b6ad79253d50a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java @@ -28,8 +28,10 @@ import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.UserException; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.resource.Tag; import org.apache.doris.utframe.UtFrameUtils; +import com.google.common.collect.Maps; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -37,6 +39,7 @@ import java.io.File; import java.util.HashSet; +import java.util.Map; import java.util.Set; import java.util.UUID; @@ -783,4 +786,19 @@ public void testCreateTableWithForceReplica() throws DdlException { Assert.assertEquals(1, tb.getPartitionInfo().getReplicaAllocation(p1.getId()).getTotalReplicaNum()); Assert.assertEquals(1, tb.getTableProperty().getReplicaAllocation().getTotalReplicaNum()); } + + @Test + public void testCreateTableDetailMsg() throws Exception { + Map allocMap = Maps.newHashMap(); + allocMap.put(Tag.create(Tag.TYPE_LOCATION, "group_a"), (short) 6); + Assert.assertEquals(" Backends details: backends with tag {\"location\" : \"group_a\"} is [], ", + Env.getCurrentSystemInfo().getDetailsForCreateReplica(new ReplicaAllocation(allocMap))); + + allocMap.clear(); + allocMap.put(Tag.create(Tag.TYPE_LOCATION, new String(Tag.VALUE_DEFAULT_TAG)), (short) 6); + String msg = Env.getCurrentSystemInfo().getDetailsForCreateReplica(new ReplicaAllocation(allocMap)); + Assert.assertTrue("msg: " + msg, msg.contains("Backends details: backends with tag {\"location\" : \"default\"} is [[backendId=") + && msg.contains("hdd disks count={ok=1,}, ssd disk count={}]")); + } + }