Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use milliseconds to express TTL #12757

Merged
merged 8 commits into from
Jun 24, 2024
Merged

Use milliseconds to express TTL #12757

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8660b86
use ms unit with ttl
choubenson Jun 18, 2024
992195f
fix IT
choubenson Jun 18, 2024
a782704
fix IT
choubenson Jun 18, 2024
eeaaf02
merge master
choubenson Jun 18, 2024
6c22fa4
fix IT
choubenson Jun 19, 2024
bcf805c
remove ttl column when showing databases
choubenson Jun 19, 2024
9574e6f
add log when integer overflow in CommonDateTimeUtils
choubenson Jun 19, 2024
23da86c
Merge branch 'master' into TTLUnit
choubenson Jun 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRestServiceIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import java.util.List;
import java.util.Map;

import static org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant.COLUMN_TTL;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
Expand Down Expand Up @@ -1063,7 +1064,7 @@ public void showAllTTL(CloseableHttpClient httpClient) {
new ArrayList<Object>() {
{
add("Device");
add("TTL");
add(COLUMN_TTL);
}
};
List<Object> values1 =
Expand Down Expand Up @@ -1093,7 +1094,6 @@ public void showStorageGroup(CloseableHttpClient httpClient) {
new ArrayList<Object>() {
{
add("Database");
add("TTL");
add("SchemaReplicationFactor");
add("DataReplicationFactor");
add("TimePartitionInterval");
Expand Down Expand Up @@ -1307,7 +1307,7 @@ public void showDevices(CloseableHttpClient httpClient) {
add("Device");
add("IsAligned");
add("Template");
add("TTL");
add(COLUMN_TTL);
}
};
List<Object> values1 =
Expand Down Expand Up @@ -1345,7 +1345,7 @@ public void showDevicesWithStroage(CloseableHttpClient httpClient) {
add("Database");
add("IsAligned");
add("Template");
add("TTL");
add(COLUMN_TTL);
}
};
List<Object> values1 =
Expand Down Expand Up @@ -1715,7 +1715,7 @@ public void showAllTTLV2(CloseableHttpClient httpClient) {
new ArrayList<Object>() {
{
add("Device");
add("TTL");
add(COLUMN_TTL);
}
};
List<Object> values1 =
Expand Down Expand Up @@ -1745,7 +1745,6 @@ public void showStorageGroupV2(CloseableHttpClient httpClient) {
new ArrayList<Object>() {
{
add("Database");
add("TTL");
add("SchemaReplicationFactor");
add("DataReplicationFactor");
add("TimePartitionInterval");
Expand Down Expand Up @@ -1959,7 +1958,7 @@ public void showDevicesV2(CloseableHttpClient httpClient) {
add("Device");
add("IsAligned");
add("Template");
add("TTL");
add(COLUMN_TTL);
}
};
List<Object> values1 =
Expand Down Expand Up @@ -1998,7 +1997,7 @@ public void showDevicesWithStroageV2(CloseableHttpClient httpClient) {
add("Database");
add("IsAligned");
add("Template");
add("TTL");
add(COLUMN_TTL);
}
};
List<Object> values1 =
Expand Down
2 changes: 1 addition & 1 deletion integration-test/src/test/java/org/apache/iotdb/db/it/auth/IoTDBSeriesPermissionIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ private void testReadSchema() {
showStorageGroupsColumnHeaders.stream()
.map(ColumnHeader::getColumnName)
.toArray(String[]::new),
new String[] {"root.test,10000,1,1,604800000,"},
new String[] {"root.test,1,1,604800000,"},
"test1",
"test123");
resultSetEqualTest(
Expand Down
2 changes: 1 addition & 1 deletion integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeDataSinkIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ private void testReceiverAutoCreate(final Map<String, String> extractorAttribute
TestUtils.assertDataEventuallyOnEnv(
receiverEnv,
"show devices root.ln.wf01.wt02",
"Device,IsAligned,Template,TTL,",
"Device,IsAligned,Template,TTL(ms),",
Collections.singleton("root.ln.wf01.wt02,true,null,INF,"));
}
}
Expand Down
2 changes: 1 addition & 1 deletion integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeExtractorIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,7 +497,7 @@ public void testExtractorPatternMatch() throws Exception {
TestUtils.assertDataEventuallyOnEnv(
receiverEnv,
"show devices",
"Device,IsAligned,Template,TTL,",
"Device,IsAligned,Template,TTL(ms),",
new HashSet<>(
Arrays.asList(
"root.nonAligned.1TS,false,null,INF,",
Expand Down
6 changes: 3 additions & 3 deletions ...gration-test/src/test/java/org/apache/iotdb/pipe/it/manual/IoTDBPipeMetaHistoricalIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,10 +150,10 @@ public void testTemplateInclusion() throws Exception {
TestUtils.assertDataEventuallyOnEnv(
receiverEnv,
"show databases",
"Database,TTL,SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,",
"Database,SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,",
// Receiver's SchemaReplicationFactor/DataReplicationFactor shall be 3/2 regardless of the
// sender
Collections.singleton("root.ln,3600000,3,2,604800000,"));
Collections.singleton("root.ln,3,2,604800000,"));
TestUtils.assertDataEventuallyOnEnv(
receiverEnv,
"select * from root.**",
Expand Down Expand Up @@ -245,7 +245,7 @@ public void testAuthInclusion() throws Exception {
TestUtils.assertDataAlwaysOnEnv(
receiverEnv,
"show databases",
"Database,TTL,SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,",
"Database,SchemaReplicationFactor,DataReplicationFactor,TimePartitionInterval,",
Collections.emptySet());
TestUtils.assertDataAlwaysOnEnv(
receiverEnv, "select * from root.**", "Time", Collections.emptySet());
Expand Down
12 changes: 0 additions & 12 deletions iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/TTLManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.confignode.consensus.request.read.ttl.ShowTTLPlan;
import org.apache.iotdb.confignode.consensus.request.write.database.DatabaseSchemaPlan;
Expand Down Expand Up @@ -63,10 +62,6 @@ public TSStatus setTTL(DatabaseSchemaPlan databaseSchemaPlan, final boolean isGe
errorStatus.setMessage("The TTL should be positive.");
return errorStatus;
}
ttl =
CommonDateTimeUtils.convertMilliTimeWithPrecision(
ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
ttl = ttl <= 0 ? Long.MAX_VALUE : ttl;
SetTTLPlan setTTLPlan =
new SetTTLPlan(
PathUtils.splitPathToDetachedNodes(databaseSchemaPlan.getSchema().getName()), ttl);
Expand All @@ -93,13 +88,6 @@ public TSStatus setTTL(SetTTLPlan setTTLPlan, final boolean isGeneratedByPipe) {
// if path matches database, then set both path and path.**
setTTLPlan.setDataBase(configManager.getPartitionManager().isDatabaseExist(path.getFullPath()));

long ttl =
CommonDateTimeUtils.convertMilliTimeWithPrecision(
setTTLPlan.getTTL(),
CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
ttl = ttl <= 0 ? Long.MAX_VALUE : ttl;
setTTLPlan.setTTL(ttl);

return configManager.getProcedureManager().setTTL(setTTLPlan, isGeneratedByPipe);
}

Expand Down
5 changes: 0 additions & 5 deletions ...ignode/src/main/java/org/apache/iotdb/confignode/manager/schema/ClusterSchemaManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.apache.iotdb.commons.path.PathPatternTree;
import org.apache.iotdb.commons.schema.SchemaConstant;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.commons.utils.StatusUtils;
import org.apache.iotdb.confignode.client.CnToDnRequestType;
Expand Down Expand Up @@ -401,10 +400,6 @@ public Map<String, Long> getTTLInfoForUpgrading() {
if (ttl <= 0 || ttl == CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs()) {
continue;
}
ttl =
CommonDateTimeUtils.convertMilliTimeWithPrecision(
ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
ttl = ttl <= 0 ? Long.MAX_VALUE : ttl;
infoMap.put(database, ttl);
} catch (DatabaseNotExistsException e) {
LOGGER.warn("Database: {} doesn't exist", databases, e);
Expand Down
3 changes: 1 addition & 2 deletions ...ode/src/main/java/org/apache/iotdb/db/queryengine/common/header/ColumnHeaderConstant.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ private ColumnHeaderConstant() {
public static final String TEMPLATE = "Template";

public static final String COUNT = "Count";
public static final String COLUMN_TTL = "TTL";
public static final String COLUMN_TTL = "TTL(ms)";
public static final String SCHEMA_REPLICATION_FACTOR = "SchemaReplicationFactor";
public static final String DATA_REPLICATION_FACTOR = "DataReplicationFactor";
public static final String TIME_PARTITION_INTERVAL = "TimePartitionInterval";
Expand Down Expand Up @@ -252,7 +252,6 @@ private ColumnHeaderConstant() {
public static final List<ColumnHeader> showStorageGroupsColumnHeaders =
ImmutableList.of(
new ColumnHeader(DATABASE, TSDataType.TEXT),
new ColumnHeader(COLUMN_TTL, TSDataType.TEXT),
new ColumnHeader(SCHEMA_REPLICATION_FACTOR, TSDataType.INT32),
new ColumnHeader(DATA_REPLICATION_FACTOR, TSDataType.INT32),
new ColumnHeader(TIME_PARTITION_INTERVAL, TSDataType.INT64));
Expand Down
2 changes: 1 addition & 1 deletion .../org/apache/iotdb/db/queryengine/execution/operator/schema/source/DeviceSchemaSource.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ public void transformToTsBlockColumns(
.getColumnBuilder(0)
.writeBinary(new Binary(device.getFullPath(), TSFileConfig.STRING_CHARSET));
int templateId = device.getTemplateId();
long ttl = DataNodeTTLCache.getInstance().getTTL(device.getPartialPath().getNodes());
long ttl = DataNodeTTLCache.getInstance().getTTLInMS(device.getPartialPath().getNodes());
// TODO: make it more readable, like "30 days" or "10 hours"
String ttlStr = ttl == Long.MAX_VALUE ? IoTDBConstant.TTL_INFINITE : String.valueOf(ttl);
if (hasSgCol) {
Expand Down
31 changes: 28 additions & 3 deletions ...main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeTTLCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@
*/
package org.apache.iotdb.db.queryengine.plan.analyze.cache.schema;

import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.schema.ttl.TTLCache;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.commons.utils.PathUtils;
import org.apache.iotdb.commons.utils.TestOnly;

Expand Down Expand Up @@ -82,16 +84,36 @@ public long getTTL(IDeviceID deviceID) {
}
}

/** Get ttl with time precision conversion. */
public long getTTL(String path) throws IllegalPathException {
lock.readLock().lock();
try {
return ttlCache.getClosestTTL(PathUtils.splitPathToDetachedNodes(path));
long ttl = ttlCache.getClosestTTL(PathUtils.splitPathToDetachedNodes(path));
return ttl == Long.MAX_VALUE
? ttl
: CommonDateTimeUtils.convertMilliTimeWithPrecision(
ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
} finally {
lock.readLock().unlock();
}
}

/** Get ttl with time precision conversion. */
public long getTTL(String[] path) {
lock.readLock().lock();
try {
long ttl = ttlCache.getClosestTTL(path);
return ttl == Long.MAX_VALUE
? ttl
: CommonDateTimeUtils.convertMilliTimeWithPrecision(
ttl, CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
} finally {
lock.readLock().unlock();
}
}

/** Get ttl without time precision conversion. */
public long getTTLInMS(String[] path) {
lock.readLock().lock();
try {
return ttlCache.getClosestTTL(path);
Expand All @@ -100,8 +122,11 @@ public long getTTL(String[] path) {
}
}

/** Get ttl of one specific path node. If this node does not set ttl, then return -1. */
public long getNodeTTL(String path) throws IllegalPathException {
/**
* Get ttl of one specific path node without time precision conversion. If this node does not set
* ttl, then return -1.
*/
public long getNodeTTLInMS(String path) throws IllegalPathException {
lock.readLock().lock();
try {
return ttlCache.getLastNodeTTL(PathUtils.splitPathToDetachedNodes(path));
Expand Down
37 changes: 10 additions & 27 deletions ...n/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/ShowDatabaseStatement.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,14 @@

package org.apache.iotdb.db.queryengine.plan.statement.metadata;

import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.schema.ttl.TTLCache;
import org.apache.iotdb.confignode.rpc.thrift.TDatabaseInfo;
import org.apache.iotdb.db.queryengine.common.header.ColumnHeader;
import org.apache.iotdb.db.queryengine.common.header.ColumnHeaderConstant;
import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory;
import org.apache.iotdb.db.queryengine.plan.analyze.QueryType;
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
import org.apache.iotdb.db.queryengine.plan.statement.IConfigStatement;
import org.apache.iotdb.db.queryengine.plan.statement.StatementVisitor;
Expand Down Expand Up @@ -93,35 +90,21 @@ public void buildTSBlock(
for (Map.Entry<String, TDatabaseInfo> entry : storageGroupInfoMap.entrySet()) {
String storageGroup = entry.getKey();
TDatabaseInfo storageGroupInfo = entry.getValue();
long ttl =
DataNodeTTLCache.getInstance()
.getNodeTTL(
storageGroup
+ IoTDBConstant.PATH_SEPARATOR
+ IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD);
String ttlStr = String.valueOf(ttl);
if (ttl == TTLCache.NULL_TTL) {
ttlStr = "null";
} else if (ttl == Long.MAX_VALUE) {
ttlStr = IoTDBConstant.TTL_INFINITE;
}

builder.getTimeColumnBuilder().writeLong(0L);
builder
.getColumnBuilder(0)
.writeBinary(new Binary(storageGroup, TSFileConfig.STRING_CHARSET));

builder.getColumnBuilder(1).writeBinary(new Binary(ttlStr, TSFileConfig.STRING_CHARSET));

builder.getColumnBuilder(2).writeInt(storageGroupInfo.getSchemaReplicationFactor());
builder.getColumnBuilder(3).writeInt(storageGroupInfo.getDataReplicationFactor());
builder.getColumnBuilder(4).writeLong(storageGroupInfo.getTimePartitionInterval());
builder.getColumnBuilder(1).writeInt(storageGroupInfo.getSchemaReplicationFactor());
builder.getColumnBuilder(2).writeInt(storageGroupInfo.getDataReplicationFactor());
builder.getColumnBuilder(3).writeLong(storageGroupInfo.getTimePartitionInterval());
if (isDetailed) {
builder.getColumnBuilder(5).writeInt(storageGroupInfo.getSchemaRegionNum());
builder.getColumnBuilder(6).writeInt(storageGroupInfo.getMinSchemaRegionNum());
builder.getColumnBuilder(7).writeInt(storageGroupInfo.getMaxSchemaRegionNum());
builder.getColumnBuilder(8).writeInt(storageGroupInfo.getDataRegionNum());
builder.getColumnBuilder(9).writeInt(storageGroupInfo.getMinDataRegionNum());
builder.getColumnBuilder(10).writeInt(storageGroupInfo.getMaxDataRegionNum());
builder.getColumnBuilder(4).writeInt(storageGroupInfo.getSchemaRegionNum());
builder.getColumnBuilder(5).writeInt(storageGroupInfo.getMinSchemaRegionNum());
builder.getColumnBuilder(6).writeInt(storageGroupInfo.getMaxSchemaRegionNum());
builder.getColumnBuilder(7).writeInt(storageGroupInfo.getDataRegionNum());
builder.getColumnBuilder(8).writeInt(storageGroupInfo.getMinDataRegionNum());
builder.getColumnBuilder(9).writeInt(storageGroupInfo.getMaxDataRegionNum());
}
builder.declarePosition();
}
Expand Down
1 change: 0 additions & 1 deletion iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,6 @@ private IoTDBConstant() {}
public static final String STATEMENT = "statement";

public static final String COLUMN_DATABASE = "database";
public static final String COLUMN_TTL = "ttl";

public static final String COLUMN_FUNCTION_NAME = "function name";
public static final String COLUMN_FUNCTION_TYPE = "function type";
Expand Down
18 changes: 5 additions & 13 deletions iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/ttl/TTLCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.commons.utils.PathUtils;

import org.apache.tsfile.utils.ReadWriteIOUtils;
Expand All @@ -47,12 +46,9 @@ public class TTLCache {

public TTLCache() {
ttlCacheTree = new CacheNode(IoTDBConstant.PATH_ROOT);
long defaultTTL =
CommonDateTimeUtils.convertMilliTimeWithPrecision(
CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs(),
CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
defaultTTL = defaultTTL <= 0 ? Long.MAX_VALUE : defaultTTL;
ttlCacheTree.addChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD, defaultTTL);
ttlCacheTree.addChild(
IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD,
CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs());
ttlCount = 1;
}

Expand Down Expand Up @@ -93,9 +89,7 @@ public void unsetTTL(String[] nodes) {
if (nodes[0].equals(IoTDBConstant.PATH_ROOT)
&& nodes[1].equals(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD)) {
ttlCacheTree.getChild(IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD).ttl =
CommonDateTimeUtils.convertMilliTimeWithPrecision(
CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs(),
CommonDescriptor.getInstance().getConfig().getTimestampPrecision());
CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs();
return;
}
}
Expand Down Expand Up @@ -231,9 +225,7 @@ public void clear() {
ttlCacheTree.removeAllChildren();
ttlCacheTree.addChild(
IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD,
CommonDateTimeUtils.convertMilliTimeWithPrecision(
CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs(),
CommonDescriptor.getInstance().getConfig().getTimestampPrecision()));
CommonDescriptor.getInstance().getConfig().getDefaultTTLInMs());
}

static class CacheNode {
Expand Down
Loading
Loading