Use milliseconds to express TTL (#12757)

* use ms unit with ttl * fix IT * fix IT * fix IT * remove ttl column when showing databases * add log when integer overflow in CommonDateTimeUtils
apache · Jun 24, 2024 · cbdade4 · cbdade4
1 parent a96349e
commit cbdade4
Show file tree

Hide file tree

Showing 14 changed files with 68 additions and 78 deletions.
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java
@@ -53,6 +53,7 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.COLUMN_TTL;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -1063,7 +1064,7 @@ public void showAllTTL(CloseableHttpClient httpClient) {
         new ArrayList<Object>() {
           {
             add("Device");
-            add("TTL");
+            add(COLUMN_TTL);
           }
         };
     List<Object> values1 =
@@ -1093,7 +1094,6 @@ public void showStorageGroup(CloseableHttpClient httpClient) {
         new ArrayList<Object>() {
           {
             add("Database");
-            add("TTL");
             add("SchemaReplicationFactor");
             add("DataReplicationFactor");
             add("TimePartitionInterval");
@@ -1307,7 +1307,7 @@ public void showDevices(CloseableHttpClient httpClient) {
             add("Device");
             add("IsAligned");
             add("Template");
-            add("TTL");
+            add(COLUMN_TTL);
           }
         };
     List<Object> values1 =
@@ -1345,7 +1345,7 @@ public void showDevicesWithStroage(CloseableHttpClient httpClient) {
             add("Database");
             add("IsAligned");
             add("Template");
-            add("TTL");
+            add(COLUMN_TTL);
           }
         };
     List<Object> values1 =
@@ -1715,7 +1715,7 @@ public void showAllTTLV2(CloseableHttpClient httpClient) {
         new ArrayList<Object>() {
           {
             add("Device");
-            add("TTL");
+            add(COLUMN_TTL);
           }
         };
     List<Object> values1 =
@@ -1745,7 +1745,6 @@ public void showStorageGroupV2(CloseableHttpClient httpClient) {
         new ArrayList<Object>() {
           {
             add("Database");
-            add("TTL");
             add("SchemaReplicationFactor");
             add("DataReplicationFactor");
             add("TimePartitionInterval");
@@ -1959,7 +1958,7 @@ public void showDevicesV2(CloseableHttpClient httpClient) {
             add("Device");
             add("IsAligned");
             add("Template");
-            add("TTL");
+            add(COLUMN_TTL);
           }
         };
     List<Object> values1 =
@@ -1998,7 +1997,7 @@ public void showDevicesWithStroageV2(CloseableHttpClient httpClient) {
             add("Database");
             add("IsAligned");
             add("Template");
-            add("TTL");
+            add(COLUMN_TTL);
           }
         };
     List<Object> values1 =

diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSeriesPermissionIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSeriesPermissionIT.java
@@ -163,7 +163,7 @@ private void testReadSchema() {
         showStorageGroupsColumnHeaders.stream()
             .map(ColumnHeader::getColumnName)
             .toArray(String[]::new),
-        new String[] {"root.test,10000,1,1,604800000,"},
+        new String[] {"root.test,1,1,604800000,"},
         "test1",
         "test123");
     resultSetEqualTest(

diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java
@@ -249,7 +249,7 @@ private void testReceiverAutoCreate(final Map<String, String> extractorAttribute
       TestUtils.assertDataEventuallyOnEnv(
           receiverEnv,
           "show devices root.ln.wf01.wt02",
-          "Device,IsAligned,Template,TTL,",
+          "Device,IsAligned,Template,TTL(ms),",
           Collections.singleton("root.ln.wf01.wt02,true,null,INF,"));
     }
   }

diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java
@@ -497,7 +497,7 @@ public void testExtractorPatternMatch() throws Exception {
       TestUtils.assertDataEventuallyOnEnv(
           receiverEnv,
           "show devices",
-          "Device,IsAligned,Template,TTL,",
+          "Device,IsAligned,Template,TTL(ms),",
           new HashSet<>(
               Arrays.asList(
                   "root.nonAligned.1TS,false,null,INF,",

diff --git a/...gration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java b/...gration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java
@@ -150,10 +150,10 @@ public void testTemplateInclusion() throws Exception {
       TestUtils.assertDataEventuallyOnEnv(
           receiverEnv,
           "show databases",
-          "Database,TTL,SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,",
+          "Database,SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,",
           // Receiver's SchemaReplicationFactor/DataReplicationFactor shall be 3/2 regardless of the
           // sender
-          Collections.singleton("root.ln,3600000,3,2,604800000,"));
+          Collections.singleton("root.ln,3,2,604800000,"));
       TestUtils.assertDataEventuallyOnEnv(
           receiverEnv,
           "select * from root.**",
@@ -245,7 +245,7 @@ public void testAuthInclusion() throws Exception {
       TestUtils.assertDataAlwaysOnEnv(
           receiverEnv,
           "show databases",
-          "Database,TTL,SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,",
+          "Database,SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,",
           Collections.emptySet());
       TestUtils.assertDataAlwaysOnEnv(
           receiverEnv, "select * from root.**", "Time", Collections.emptySet());

diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java
@@ -22,7 +22,6 @@
 import org.apache.iotdb.commons.conf.CommonDescriptor;
 import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
 import org.apache.iotdb.commons.utils.PathUtils;
 import org.apache.iotdb.confignode.consensus.request.read.ttl.ShowTTLPlan;
 import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan;
@@ -63,10 +62,6 @@ public TSStatus setTTL(DatabaseSchemaPlan databaseSchemaPlan, final boolean isGe
       errorStatus.setMessage("The TTL should be positive.");
       return errorStatus;
     }
-    ttl =
-        CommonDateTimeUtils.convertMilliTimeWithPrecision(
-            ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
-    ttl = ttl <= 0 ? Long.MAX_VALUE : ttl;
     SetTTLPlan setTTLPlan =
         new SetTTLPlan(
             PathUtils.splitPathToDetachedNodes(databaseSchemaPlan.getSchema().getName()), ttl);
@@ -93,13 +88,6 @@ public TSStatus setTTL(SetTTLPlan setTTLPlan, final boolean isGeneratedByPipe) {
     // if path matches database, then set both path and path.**
     setTTLPlan.setDataBase(configManager.getPartitionManager().isDatabaseExist(path.getFullPath()));
 
-    long ttl =
-        CommonDateTimeUtils.convertMilliTimeWithPrecision(
-            setTTLPlan.getTTL(),
-            CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
-    ttl = ttl <= 0 ? Long.MAX_VALUE : ttl;
-    setTTLPlan.setTTL(ttl);
-
     return configManager.getProcedureManager().setTTL(setTTLPlan, isGeneratedByPipe);
   }
 

diff --git a/...ignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java b/...ignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
@@ -29,7 +29,6 @@
 import org.apache.iotdb.commons.path.PathPatternTree;
 import org.apache.iotdb.commons.schema.SchemaConstant;
 import org.apache.iotdb.commons.service.metric.MetricService;
-import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
 import org.apache.iotdb.commons.utils.PathUtils;
 import org.apache.iotdb.commons.utils.StatusUtils;
 import org.apache.iotdb.confignode.client.CnToDnRequestType;
@@ -401,10 +400,6 @@ public Map<String, Long> getTTLInfoForUpgrading() {
         if (ttl <= 0 || ttl == CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs()) {
           continue;
         }
-        ttl =
-            CommonDateTimeUtils.convertMilliTimeWithPrecision(
-                ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
-        ttl = ttl <= 0 ? Long.MAX_VALUE : ttl;
         infoMap.put(database, ttl);
       } catch (DatabaseNotExistsException e) {
         LOGGER.warn("Database: {} doesn't exist", databases, e);

diff --git a/...ode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java b/...ode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java
@@ -52,7 +52,7 @@ private ColumnHeaderConstant() {
   public static final String TEMPLATE = "Template";
 
   public static final String COUNT = "Count";
-  public static final String COLUMN_TTL = "TTL";
+  public static final String COLUMN_TTL = "TTL(ms)";
   public static final String SCHEMA_REPLICATION_FACTOR = "SchemaReplicationFactor";
   public static final String DATA_REPLICATION_FACTOR = "DataReplicationFactor";
   public static final String TIME_PARTITION_INTERVAL = "TimePartitionInterval";
@@ -252,7 +252,6 @@ private ColumnHeaderConstant() {
   public static final List<ColumnHeader> showStorageGroupsColumnHeaders =
       ImmutableList.of(
           new ColumnHeader(DATABASE, TSDataType.TEXT),
-          new ColumnHeader(COLUMN_TTL, TSDataType.TEXT),
           new ColumnHeader(SCHEMA_REPLICATION_FACTOR, TSDataType.INT32),
           new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32),
           new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64));

diff --git a/.../org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceSchemaSource.java b/.../org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceSchemaSource.java
@@ -101,7 +101,7 @@ public void transformToTsBlockColumns(
         .getColumnBuilder(0)
         .writeBinary(new Binary(device.getFullPath(), TSFileConfig.STRING_CHARSET));
     int templateId = device.getTemplateId();
-    long ttl = DataNodeTTLCache.getInstance().getTTL(device.getPartialPath().getNodes());
+    long ttl = DataNodeTTLCache.getInstance().getTTLInMS(device.getPartialPath().getNodes());
     // TODO: make it more readable, like "30 days" or "10 hours"
     String ttlStr = ttl == Long.MAX_VALUE ? IoTDBConstant.TTL_INFINITE : String.valueOf(ttl);
     if (hasSgCol) {

diff --git a/...main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java b/...main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
@@ -18,8 +18,10 @@
  */
 package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema;
 
+import org.apache.iotdb.commons.conf.CommonDescriptor;
 import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.schema.ttl.TTLCache;
+import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
 import org.apache.iotdb.commons.utils.PathUtils;
 import org.apache.iotdb.commons.utils.TestOnly;
 
@@ -82,16 +84,36 @@ public long getTTL(IDeviceID deviceID) {
     }
   }
 
+  /** Get ttl with time precision conversion. */
   public long getTTL(String path) throws IllegalPathException {
     lock.readLock().lock();
     try {
-      return ttlCache.getClosestTTL(PathUtils.splitPathToDetachedNodes(path));
+      long ttl = ttlCache.getClosestTTL(PathUtils.splitPathToDetachedNodes(path));
+      return ttl == Long.MAX_VALUE
+          ? ttl
+          : CommonDateTimeUtils.convertMilliTimeWithPrecision(
+              ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
     } finally {
       lock.readLock().unlock();
     }
   }
 
+  /** Get ttl with time precision conversion. */
   public long getTTL(String[] path) {
+    lock.readLock().lock();
+    try {
+      long ttl = ttlCache.getClosestTTL(path);
+      return ttl == Long.MAX_VALUE
+          ? ttl
+          : CommonDateTimeUtils.convertMilliTimeWithPrecision(
+              ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
+    } finally {
+      lock.readLock().unlock();
+    }
+  }
+
+  /** Get ttl without time precision conversion. */
+  public long getTTLInMS(String[] path) {
     lock.readLock().lock();
     try {
       return ttlCache.getClosestTTL(path);
@@ -100,8 +122,11 @@ public long getTTL(String[] path) {
     }
   }
 
-  /** Get ttl of one specific path node. If this node does not set ttl, then return -1. */
-  public long getNodeTTL(String path) throws IllegalPathException {
+  /**
+   * Get ttl of one specific path node without time precision conversion. If this node does not set
+   * ttl, then return -1.
+   */
+  public long getNodeTTLInMS(String path) throws IllegalPathException {
     lock.readLock().lock();
     try {
       return ttlCache.getLastNodeTTL(PathUtils.splitPathToDetachedNodes(path));

diff --git a/...n/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java b/...n/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java
@@ -19,17 +19,14 @@
 
 package org.apache.iotdb.db.queryengine.plan.statement.metadata;
 
-import org.apache.iotdb.commons.conf.IoTDBConstant;
 import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.commons.schema.ttl.TTLCache;
 import org.apache.iotdb.confignode.rpc.thrift.TDatabaseInfo;
 import org.apache.iotdb.db.queryengine.common.header.ColumnHeader;
 import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
 import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
 import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory;
 import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
-import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
 import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
 import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement;
 import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
@@ -93,35 +90,21 @@ public void buildTSBlock(
     for (Map.Entry<String, TDatabaseInfo> entry : storageGroupInfoMap.entrySet()) {
       String storageGroup = entry.getKey();
       TDatabaseInfo storageGroupInfo = entry.getValue();
-      long ttl =
-          DataNodeTTLCache.getInstance()
-              .getNodeTTL(
-                  storageGroup
-                      + IoTDBConstant.PATH_SEPARATOR
-                      + IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD);
-      String ttlStr = String.valueOf(ttl);
-      if (ttl == TTLCache.NULL_TTL) {
-        ttlStr = "null";
-      } else if (ttl == Long.MAX_VALUE) {
-        ttlStr = IoTDBConstant.TTL_INFINITE;
-      }
+
       builder.getTimeColumnBuilder().writeLong(0L);
       builder
           .getColumnBuilder(0)
           .writeBinary(new Binary(storageGroup, TSFileConfig.STRING_CHARSET));
-
-      builder.getColumnBuilder(1).writeBinary(new Binary(ttlStr, TSFileConfig.STRING_CHARSET));
-
-      builder.getColumnBuilder(2).writeInt(storageGroupInfo.getSchemaReplicationFactor());
-      builder.getColumnBuilder(3).writeInt(storageGroupInfo.getDataReplicationFactor());
-      builder.getColumnBuilder(4).writeLong(storageGroupInfo.getTimePartitionInterval());
+      builder.getColumnBuilder(1).writeInt(storageGroupInfo.getSchemaReplicationFactor());
+      builder.getColumnBuilder(2).writeInt(storageGroupInfo.getDataReplicationFactor());
+      builder.getColumnBuilder(3).writeLong(storageGroupInfo.getTimePartitionInterval());
       if (isDetailed) {
-        builder.getColumnBuilder(5).writeInt(storageGroupInfo.getSchemaRegionNum());
-        builder.getColumnBuilder(6).writeInt(storageGroupInfo.getMinSchemaRegionNum());
-        builder.getColumnBuilder(7).writeInt(storageGroupInfo.getMaxSchemaRegionNum());
-        builder.getColumnBuilder(8).writeInt(storageGroupInfo.getDataRegionNum());
-        builder.getColumnBuilder(9).writeInt(storageGroupInfo.getMinDataRegionNum());
-        builder.getColumnBuilder(10).writeInt(storageGroupInfo.getMaxDataRegionNum());
+        builder.getColumnBuilder(4).writeInt(storageGroupInfo.getSchemaRegionNum());
+        builder.getColumnBuilder(5).writeInt(storageGroupInfo.getMinSchemaRegionNum());
+        builder.getColumnBuilder(6).writeInt(storageGroupInfo.getMaxSchemaRegionNum());
+        builder.getColumnBuilder(7).writeInt(storageGroupInfo.getDataRegionNum());
+        builder.getColumnBuilder(8).writeInt(storageGroupInfo.getMinDataRegionNum());
+        builder.getColumnBuilder(9).writeInt(storageGroupInfo.getMaxDataRegionNum());
       }
       builder.declarePosition();
     }

diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -171,7 +171,6 @@ private IoTDBConstant() {}
   public static final String STATEMENT = "statement";
 
   public static final String COLUMN_DATABASE = "database";
-  public static final String COLUMN_TTL = "ttl";
 
   public static final String COLUMN_FUNCTION_NAME = "function name";
   public static final String COLUMN_FUNCTION_TYPE = "function type";

diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java
@@ -21,7 +21,6 @@
 import org.apache.iotdb.commons.conf.CommonDescriptor;
 import org.apache.iotdb.commons.conf.IoTDBConstant;
 import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
 import org.apache.iotdb.commons.utils.PathUtils;
 
 import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -47,12 +46,9 @@ public class TTLCache {
 
   public TTLCache() {
     ttlCacheTree = new CacheNode(IoTDBConstant.PATH_ROOT);
-    long defaultTTL =
-        CommonDateTimeUtils.convertMilliTimeWithPrecision(
-            CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs(),
-            CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
-    defaultTTL = defaultTTL <= 0 ? Long.MAX_VALUE : defaultTTL;
-    ttlCacheTree.addChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD, defaultTTL);
+    ttlCacheTree.addChild(
+        IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD,
+        CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs());
     ttlCount = 1;
   }
 
@@ -93,9 +89,7 @@ public void unsetTTL(String[] nodes) {
       if (nodes[0].equals(IoTDBConstant.PATH_ROOT)
           && nodes[1].equals(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD)) {
         ttlCacheTree.getChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD).ttl =
-            CommonDateTimeUtils.convertMilliTimeWithPrecision(
-                CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs(),
-                CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
+            CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs();
         return;
       }
     }
@@ -231,9 +225,7 @@ public void clear() {
     ttlCacheTree.removeAllChildren();
     ttlCacheTree.addChild(
         IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD,
-        CommonDateTimeUtils.convertMilliTimeWithPrecision(
-            CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs(),
-            CommonDescriptor.getInstance().getConfig().getTimestampPrecision()));
+        CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs());
   }
 
   static class CacheNode {