diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBFillTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBFillTableIT.java new file mode 100644 index 000000000000..512dc0958a25 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBFillTableIT.java @@ -0,0 +1,583 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.query.recent; + +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import static org.apache.iotdb.db.it.utils.TestUtils.prepareTableData; +import static org.apache.iotdb.db.it.utils.TestUtils.tableAssertTestFail; +import static org.apache.iotdb.db.it.utils.TestUtils.tableResultSetEqualTest; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBFillTableIT { + private static final String DATABASE_NAME = "test"; + private static final String[] createSqls = + new String[] { + "CREATE DATABASE " + DATABASE_NAME, + "USE " + DATABASE_NAME, + "CREATE TABLE table1(device_id STRING ID, s1 INT32 MEASUREMENT, s2 INT64 MEASUREMENT, s3 FLOAT MEASUREMENT, s4 DOUBLE MEASUREMENT, s5 BOOLEAN MEASUREMENT, s6 TEXT MEASUREMENT, s7 STRING MEASUREMENT, s8 BLOB MEASUREMENT, s9 TIMESTAMP MEASUREMENT, s10 DATE MEASUREMENT)", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5,s6,s7,s8,s9,s10) " + + " values(1, 'd1', 1, 11, 1.1, 11.1, true, 'text1', 'string1', X'cafebabe01', 1, '2024-10-01')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5) " + + " values(2, 'd1', 2, 22, 2.2, 22.2, false)", + "INSERT INTO table1(time,device_id,s6,s7,s8,s9,s10) " + + " values(3, 'd1', 'text3', 'string3', X'cafebabe03', 3, '2024-10-03')", + "INSERT INTO table1(time,device_id,s6,s7,s8,s9,s10) " + + " values(4, 'd1', 'text4', 'string4', X'cafebabe04', 4, '2024-10-04')", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5) " + + " values(5, 'd1', 5, 55, 5.5, 55.5, false)", + "INSERT INTO table1(time,device_id,s1,s2,s3,s4,s5) " + + " values(7, 'd1', 7, 77, 7.7, 77.7, true)", + "INSERT INTO table1(time,device_id,s6,s7,s8,s9,s10) " + + " values(8, 'd1', 'text8', 'string8', X'cafebabe08', 8, '2024-10-08')", + }; + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().getConfig().getCommonConfig().setEnableCrossSpaceCompaction(false); + EnvFactory.getEnv().initClusterEnvironment(); + prepareTableData(createSqls); + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void normalFillTest() { + + // --------------------------------- PREVIOUS FILL --------------------------------- + + // case 1: all without time filter using previous fill without timeDuration + String[] expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + String[] retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(PREVIOUS)", expectedHeader, retArray, DATABASE_NAME); + + // case 2: all with time filter using previous fill without timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 WHERE time > 1 FILL(PREVIOUS)", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: all with time filter and value filter using previous fill without timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from table1 WHERE time > 1 and time < 8 and s2 > 22 FILL(PREVIOUS)", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 4: all without time filter using previous fill with timeDuration + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(PREVIOUS, 2ms)", expectedHeader, retArray, DATABASE_NAME); + + // case 5: all without time filter using previous fill with timeDuration with helper column + // index + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(PREVIOUS(1), 2ms)", expectedHeader, retArray, DATABASE_NAME); + + // case 6: all without time filter using previous fill with timeDuration with helper column + // index + // TODO need fix + // expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", + // "s7", "s8", "s9", "s10"}; + // retArray = + // new String[] { + // + // "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + // + // "1970-01-01T00:00:00.002Z,d1,2,22,2.2,11.1,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + // + // "1970-01-01T00:00:00.003Z,d1,2,22,2.2,11.1,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + // + // "1970-01-01T00:00:00.004Z,d1,2,22,2.2,11.1,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + // + // "1970-01-01T00:00:00.005Z,d1,5,55,5.5,11.1,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + // "1970-01-01T00:00:00.007Z,d1,7,77,7.7,11.1,true,null,null,null,null,null,", + // + // "1970-01-01T00:00:00.008Z,d1,7,77,7.7,11.1,true,text8,string1,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + // }; + // tableResultSetEqualTest( + // "select * from table1 FILL(PREVIOUS(11), 2ms)", expectedHeader, retArray, + // DATABASE_NAME); + + // case7: all without time filter using previous fill with order by time desc + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(PREVIOUS(1), 2ms) order by time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case8: all without time filter using previous fill with order by value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.004Z,d1,2,22,2.2,22.2,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(PREVIOUS(1), 2ms) order by s9 desc, time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case9: all without time filter using previous fill with subQuery + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.003Z,d1,5,55,5.5,55.5,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,5,55,5.5,55.5,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from (select * from table1 order by time desc) FILL(previous, 2ms) order by time", + expectedHeader, + retArray, + DATABASE_NAME); + + // --------------------------------- LINEAR FILL --------------------------------- + // case 1: all without time filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL(LINEAR)", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 2: all with time filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 WHERE time > 1 FILL(LINEAR)", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 3: all with time filter and value filter using linear fill + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,null,null,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 WHERE time > 1 and time < 8 and s2 > 22 FILL(LINEAR)", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 5: all without time filter using linear fill with helper column + // index + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL(LINEAR(1))", + expectedHeader, + retArray, + DATABASE_NAME); + + // case 6: all without time filter using linear fill with helper column + // index + // TODO need fix + // expectedHeader = new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", + // "s7", "s8", "s9", "s10"}; + // retArray = + // new String[] { + // + // "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + // + // "1970-01-01T00:00:00.002Z,d1,2,22,2.2,11.1,false,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + // + // "1970-01-01T00:00:00.003Z,d1,2,22,2.2,11.1,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + // + // "1970-01-01T00:00:00.004Z,d1,2,22,2.2,11.1,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + // + // "1970-01-01T00:00:00.005Z,d1,5,55,5.5,11.1,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + // "1970-01-01T00:00:00.007Z,d1,7,77,7.7,11.1,true,null,null,null,null,null,", + // + // "1970-01-01T00:00:00.008Z,d1,7,77,7.7,11.1,true,text8,string1,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + // }; + // tableResultSetEqualTest( + // "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL(LINEAR(10))", + // expectedHeader, retArray, + // DATABASE_NAME); + + // case7: all without time filter using linear fill with order by time desc + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL(LINEAR) order by time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case8: all without time filter using linear fill with order by value + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + }; + tableResultSetEqualTest( + "select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 FILL(LINEAR(1)) order by s9 desc, time desc", + expectedHeader, + retArray, + DATABASE_NAME); + + // case9: all without time filter using linear fill with subQuery + expectedHeader = + new String[] {"time", "device_id", "s1", "s2", "s3", "s5", "s6", "s7", "s8", "s9", "s10"}; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,false,null,null,null,1970-01-01T00:00:00.002Z,2024-10-02,", + "1970-01-01T00:00:00.003Z,d1,3,33,3.3,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,4,44,4.4,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,false,null,null,null,1970-01-01T00:00:00.005Z,2024-10-05,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,true,null,null,null,1970-01-01T00:00:00.007Z,2024-10-07,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from (select time,device_id,s1,s2,s3,s5,s6,s7,s8,s9,s10 from table1 order by time desc) FILL(LINEAR) order by time", + expectedHeader, + retArray, + DATABASE_NAME); + + // --------------------------------- VALUE FILL --------------------------------- + // case 1: fill with integer value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.003Z,d1,100,100,100.0,100.0,true,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,100,100,100.0,100.0,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,100,100,null,1970-01-01T00:00:00.100Z,0-01-00,", + "1970-01-01T00:00:00.008Z,d1,100,100,100.0,100.0,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(100)", expectedHeader, retArray, DATABASE_NAME); + + // case 2: fill with float value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.003Z,d1,110,110,110.2,110.2,true,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,110,110,110.2,110.2,true,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,110.2,110.2,null,1970-01-01T00:00:00.110Z,0-01-10,", + "1970-01-01T00:00:00.008Z,d1,110,110,110.2,110.2,true,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(110.2)", expectedHeader, retArray, DATABASE_NAME); + + // case 3: fill with boolean value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,false,false,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,0,0,0.0,0.0,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,0,0,0.0,0.0,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,false,false,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,false,false,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,0,0,0.0,0.0,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(false)", expectedHeader, retArray, DATABASE_NAME); + + // case 3: fill with string value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,iotdb,iotdb,null,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL('iotdb')", expectedHeader, retArray, DATABASE_NAME); + + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,false,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,false,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,2018-05-06,2018-05-06,null,2018-05-06T00:00:00.000Z,2018-05-06,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,false,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL('2018-05-06')", expectedHeader, retArray, DATABASE_NAME); + + // case 4: fill with blob value + expectedHeader = + new String[] { + "time", "device_id", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10" + }; + retArray = + new String[] { + "1970-01-01T00:00:00.001Z,d1,1,11,1.1,11.1,true,text1,string1,0xcafebabe01,1970-01-01T00:00:00.001Z,2024-10-01,", + "1970-01-01T00:00:00.002Z,d1,2,22,2.2,22.2,false,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.003Z,d1,null,null,null,null,null,text3,string3,0xcafebabe03,1970-01-01T00:00:00.003Z,2024-10-03,", + "1970-01-01T00:00:00.004Z,d1,null,null,null,null,null,text4,string4,0xcafebabe04,1970-01-01T00:00:00.004Z,2024-10-04,", + "1970-01-01T00:00:00.005Z,d1,5,55,5.5,55.5,false,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.007Z,d1,7,77,7.7,77.7,true,0xcafebabe99,0xcafebabe99,0xcafebabe99,null,null,", + "1970-01-01T00:00:00.008Z,d1,null,null,null,null,null,text8,string8,0xcafebabe08,1970-01-01T00:00:00.008Z,2024-10-08,", + }; + tableResultSetEqualTest( + "select * from table1 FILL(X'cafebabe99')", expectedHeader, retArray, DATABASE_NAME); + } + + @Test + public void abNormalFillTest() { + + // --------------------------------- PREVIOUS FILL --------------------------------- + tableAssertTestFail( + "select s1 from table1 FILL(PREVIOUS(1))", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Don't need to specify helper column index while timeDuration parameter is not specified", + DATABASE_NAME); + + tableAssertTestFail( + "select s1 from table1 FILL(PREVIOUS, 2ms)", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Cannot infer the helper column for PREVIOUS FILL, there exists no column whose type is TIMESTAMP", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(PREVIOUS(1), 2ms)", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Type of helper column for PREVIOUS FILL should only be TIMESTAMP, but type of the column you specify is INT32", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(PREVIOUS(0), 2ms)", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(PREVIOUS(3), 2ms)", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": PREVIOUS FILL position 3 is not in select list", + DATABASE_NAME); + + // --------------------------------- LINEAR FILL --------------------------------- + + tableAssertTestFail( + "select s1 from table1 FILL(LINEAR)", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Cannot infer the helper column for LINEAR FILL, there exists no column whose type is TIMESTAMP", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(LINEAR(1))", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": Type of helper column for LINEAR FILL should only be TIMESTAMP, but type of the column you specify is INT32", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(LINEAR(0))", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL position 0 is not in select list", + DATABASE_NAME); + + tableAssertTestFail( + "select s1, time from table1 FILL(LINEAR(3))", + TSStatusCode.SEMANTIC_ERROR.getStatusCode() + + ": LINEAR FILL position 3 is not in select list", + DATABASE_NAME); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index d56cec281146..4c992bff86f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -195,6 +195,8 @@ public class StatementAnalyzer { private final StatementAnalyzerFactory statementAnalyzerFactory; private Analysis analysis; + + private boolean hasFillInParentScope = false; private final MPPQueryContext queryContext; private final AccessControl accessControl; @@ -508,6 +510,7 @@ protected Scope visitExplainAnalyze(ExplainAnalyze node, Optional context @Override protected Scope visitQuery(Query node, Optional context) { Scope withScope = analyzeWith(node, context); + hasFillInParentScope = node.getFill().isPresent() || hasFillInParentScope; Scope queryBodyScope = process(node.getQueryBody(), withScope); if (node.getFill().isPresent()) { @@ -521,7 +524,8 @@ protected Scope visitQuery(Query node, Optional context) { if ((queryBodyScope.getOuterQueryParent().isPresent() || !isTopLevel) && !node.getLimit().isPresent() - && !node.getOffset().isPresent()) { + && !node.getOffset().isPresent() + && hasFillInParentScope) { // not the root scope and ORDER BY is ineffective analysis.markRedundantOrderBy(node.getOrderBy().get()); warningCollector.add( @@ -696,6 +700,14 @@ private boolean tryProcessRecursiveQuery( throw new SemanticException( "immediate WITH clause in recursive query is not supported"); }); + withQuery + .getQuery() + .getFill() + .ifPresent( + orderBy -> { + throw new SemanticException( + "immediate FILL clause in recursive query is not supported"); + }); withQuery .getQuery() .getOrderBy() @@ -790,6 +802,7 @@ protected Scope visitTableSubquery(TableSubquery node, Optional scope) { protected Scope visitQuerySpecification(QuerySpecification node, Optional scope) { // TODO: extract candidate names from SELECT, WHERE, HAVING, GROUP BY and ORDER BY expressions // to pass down to analyzeFrom + hasFillInParentScope = node.getFill().isPresent() || hasFillInParentScope; Scope sourceScope = analyzeFrom(node, scope); @@ -802,9 +815,12 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional Scope outputScope = computeAndAssignOutputScope(node, scope, sourceScope); - if (node.getFill().isPresent()) { - analyzeFill(node.getFill().get(), outputScope); - } + node.getFill() + .ifPresent( + fill -> { + Scope fillScope = computeAndAssignFillScope(fill, sourceScope, outputScope); + analyzeFill(fill, fillScope); + }); List orderByExpressions = emptyList(); Optional orderByScope = Optional.empty(); @@ -816,7 +832,8 @@ protected Scope visitQuerySpecification(QuerySpecification node, Optional if ((sourceScope.getOuterQueryParent().isPresent() || !isTopLevel) && !node.getLimit().isPresent() - && !node.getOffset().isPresent()) { + && !node.getOffset().isPresent() + && hasFillInParentScope) { // not the root scope and ORDER BY is ineffective analysis.markRedundantOrderBy(orderBy); warningCollector.add( @@ -1471,6 +1488,18 @@ private Scope computeAndAssignOrderByScope(OrderBy node, Scope sourceScope, Scop return orderByScope; } + private Scope computeAndAssignFillScope(Fill node, Scope sourceScope, Scope outputScope) { + // Fill should "see" both output and FROM fields during initial analysis and + // non-aggregation query planning + Scope fillScope = + Scope.builder() + .withParent(sourceScope) + .withRelationType(outputScope.getRelationId(), outputScope.getRelationType()) + .build(); + analysis.setScope(node, fillScope); + return fillScope; + } + @Override protected Scope visitSubqueryExpression(SubqueryExpression node, Optional context) { return process(node.getQuery(), context); @@ -2098,7 +2127,7 @@ private void analyzeFill(Fill node, Scope scope) { Analysis.FillAnalysis fillAnalysis; if (node.getFillMethod() == FillPolicy.PREVIOUS) { if (node.getTimeDurationThreshold().isPresent()) { - FieldReference helperColumn = getHelperColumn(node, scope); + FieldReference helperColumn = getHelperColumn(node, scope, FillPolicy.PREVIOUS); ExpressionAnalyzer.analyzeExpression( metadata, queryContext, @@ -2131,7 +2160,7 @@ private void analyzeFill(Fill node, Scope scope) { correlationSupport); fillAnalysis = new Analysis.ValueFillAnalysis(literal); } else if (node.getFillMethod() == FillPolicy.LINEAR) { - FieldReference helperColumn = getHelperColumn(node, scope); + FieldReference helperColumn = getHelperColumn(node, scope, FillPolicy.LINEAR); ExpressionAnalyzer.analyzeExpression( metadata, queryContext, @@ -2151,19 +2180,21 @@ private void analyzeFill(Fill node, Scope scope) { analysis.setFill(node, fillAnalysis); } - private FieldReference getHelperColumn(Fill node, Scope scope) { + private FieldReference getHelperColumn(Fill node, Scope scope, FillPolicy fillMethod) { FieldReference helperColumn; if (node.getIndex().isPresent()) { long ordinal = node.getIndex().get().getParsedValue(); if (ordinal < 1 || ordinal > scope.getRelationType().getVisibleFieldCount()) { throw new SemanticException( - String.format("LINEAR FILL position %s is not in select list", ordinal)); + String.format( + "%s FILL position %s is not in select list", fillMethod.name(), ordinal)); } else if (!isTimestampType( - scope.getRelationType().getFieldByIndex((int) ordinal).getType())) { + scope.getRelationType().getFieldByIndex((int) ordinal - 1).getType())) { throw new SemanticException( String.format( - "Type of helper column for LINEAR FILL should only be TIMESTAMP, but type of the column you specify is %s", - scope.getRelationType().getFieldByIndex((int) ordinal).getType())); + "Type of helper column for %s FILL should only be TIMESTAMP, but type of the column you specify is %s", + fillMethod.name(), + scope.getRelationType().getFieldByIndex((int) ordinal - 1).getType())); } else { helperColumn = new FieldReference(toIntExact(ordinal - 1)); } @@ -2179,7 +2210,9 @@ private FieldReference getHelperColumn(Fill node, Scope scope) { } if (index == -1) { throw new SemanticException( - "Cannot infer the helper column for LINEAR FILL, there exists no column whose type is TIMESTAMP"); + String.format( + "Cannot infer the helper column for %s FILL, there exists no column whose type is TIMESTAMP", + fillMethod.name())); } helperColumn = new FieldReference(index); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToStringLiteralVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToStringLiteralVisitor.java index a64a909ec5f2..e14a58b30bc0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToStringLiteralVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToStringLiteralVisitor.java @@ -50,7 +50,7 @@ protected Binary visitLiteral(Literal node, Void context) { @Override protected Binary visitBooleanLiteral(BooleanLiteral node, Void context) { - return new Binary(BytesUtils.boolToBytes(node.getValue())); + return new Binary(String.valueOf(node.getValue()), charset); } @Override @@ -60,7 +60,7 @@ protected Binary visitLongLiteral(LongLiteral node, Void context) { @Override protected Binary visitDoubleLiteral(DoubleLiteral node, Void context) { - return new Binary(BytesUtils.doubleToBytes(node.getValue())); + return new Binary(String.valueOf(node.getValue()), charset); } @Override @@ -70,7 +70,7 @@ protected Binary visitStringLiteral(StringLiteral node, Void context) { @Override protected Binary visitBinaryLiteral(BinaryLiteral node, Void context) { - return new Binary(node.toHexString(), charset); + return new Binary(BytesUtils.parseBlobByteArrayToString(node.getValue()), charset); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToTimestampLiteralVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToTimestampLiteralVisitor.java index 9187437ecd3e..2a6dead013b4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToTimestampLiteralVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/CastToTimestampLiteralVisitor.java @@ -58,7 +58,7 @@ protected Long visitLongLiteral(LongLiteral node, Void context) { @Override protected Long visitDoubleLiteral(DoubleLiteral node, Void context) { - return null; + return (long) node.getValue(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java index 099e2aeacf05..4f8c59b5e451 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/QueryPlanner.java @@ -119,6 +119,8 @@ public QueryPlanner( public RelationPlan plan(Query query) { PlanBuilder builder = planQueryBody(query.getQueryBody()); + builder = fill(builder, query.getFill()); + // TODO result is :input[0], :input[1], :input[2] List selectExpressions = analysis.getSelectExpressions(query); List outputs = @@ -132,8 +134,6 @@ public RelationPlan plan(Query query) { builder.appendProjections( Iterables.concat(orderBy, outputs), symbolAllocator, queryContext); } - - builder = fill(builder, query.getFill()); Optional orderingScheme = orderingScheme(builder, query.getOrderBy(), analysis.getOrderByExpressions(query)); builder = sort(builder, orderingScheme); @@ -166,6 +166,23 @@ public RelationPlan plan(QuerySpecification node) { } List outputs = outputExpressions(selectExpressions); + + if (node.getFill().isPresent()) { + // Add projections for the outputs of SELECT, but stack them on top of the ones from the FROM + // clause so both are visible + // when resolving the ORDER BY clause. + builder = builder.appendProjections(outputs, symbolAllocator, queryContext); + // The new scope is the composite of the fields from the FROM and SELECT clause (local nested + // scopes). Fields from the bottom of + // the scope stack need to be placed first to match the expected layout for nested scopes. + List newFields = new ArrayList<>(builder.getTranslations().getFieldSymbolsList()); + + outputs.stream().map(builder::translate).forEach(newFields::add); + + builder = builder.withScope(analysis.getScope(node.getFill().get()), newFields); + builder = fill(builder, node.getFill()); + } + if (node.getOrderBy().isPresent()) { // ORDER BY requires outputs of SELECT to be visible. // For queries with aggregation, it also requires grouping keys and translated aggregations. @@ -201,7 +218,6 @@ public RelationPlan plan(QuerySpecification node) { Iterables.concat(orderBy, outputs), symbolAllocator, queryContext); } - builder = fill(builder, node.getFill()); Optional orderingScheme = orderingScheme(builder, node.getOrderBy(), analysis.getOrderByExpressions(node)); builder = sort(builder, orderingScheme); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java index c7e3883fc45f..cef978762eac 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/distribute/TableDistributedPlanGenerator.java @@ -50,6 +50,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; @@ -142,6 +143,9 @@ public List visitOutput(OutputNode node, PlanContext context) { @Override public List visitFill(FillNode node, PlanContext context) { + if (!(node instanceof ValueFillNode)) { + context.clearExpectedOrderingScheme(); + } List childrenNodes = node.getChild().accept(this, context); OrderingScheme childOrdering = nodeOrderingMap.get(childrenNodes.get(0).getPlanNodeId()); if (childOrdering != null) { @@ -154,6 +158,7 @@ public List visitFill(FillNode node, PlanContext context) { } node.setChild(mergeChildrenViaCollectOrMergeSort(childOrdering, childrenNodes)); + context.setHasSeenFill(true); return Collections.singletonList(node); } @@ -227,8 +232,7 @@ public List visitProject(ProjectNode node, PlanContext context) { @Override public List visitTopK(TopKNode node, PlanContext context) { - context.expectedOrderingScheme = node.getOrderingScheme(); - context.hasSortProperty = true; + context.setExpectedOrderingScheme(node.getOrderingScheme()); nodeOrderingMap.put(node.getPlanNodeId(), node.getOrderingScheme()); checkArgument( @@ -259,8 +263,7 @@ public List visitTopK(TopKNode node, PlanContext context) { @Override public List visitSort(SortNode node, PlanContext context) { - context.expectedOrderingScheme = node.getOrderingScheme(); - context.hasSortProperty = true; + context.setExpectedOrderingScheme(node.getOrderingScheme()); nodeOrderingMap.put(node.getPlanNodeId(), node.getOrderingScheme()); List childrenNodes = node.getChild().accept(this, context); @@ -285,8 +288,7 @@ public List visitSort(SortNode node, PlanContext context) { @Override public List visitStreamSort(StreamSortNode node, PlanContext context) { - context.expectedOrderingScheme = node.getOrderingScheme(); - context.hasSortProperty = true; + context.setExpectedOrderingScheme(node.getOrderingScheme()); nodeOrderingMap.put(node.getPlanNodeId(), node.getOrderingScheme()); List childrenNodes = node.getChild().accept(this, context); @@ -825,6 +827,7 @@ public List visitTableDeviceFetch( public static class PlanContext { final Map nodeDistributionMap; + boolean hasSeenFill = false; boolean hasExchangeNode = false; boolean hasSortProperty = false; OrderingScheme expectedOrderingScheme; @@ -837,5 +840,19 @@ public PlanContext() { public NodeDistribution getNodeDistribution(PlanNodeId nodeId) { return this.nodeDistributionMap.get(nodeId); } + + public void clearExpectedOrderingScheme() { + expectedOrderingScheme = null; + hasSortProperty = false; + } + + public void setExpectedOrderingScheme(OrderingScheme expectedOrderingScheme) { + this.expectedOrderingScheme = expectedOrderingScheme; + hasSortProperty = true; + } + + public void setHasSeenFill(boolean hasSeenFill) { + this.hasSeenFill = hasSeenFill; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PreviousFillNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PreviousFillNode.java index c6c58dd53247..2fb89422094e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PreviousFillNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PreviousFillNode.java @@ -104,14 +104,14 @@ protected void serializeAttributes(DataOutputStream stream) throws IOException { } public static PreviousFillNode deserialize(ByteBuffer byteBuffer) { - boolean isNull = ReadWriteIOUtils.readBool(byteBuffer); + boolean hasValue = ReadWriteIOUtils.readBool(byteBuffer); TimeDuration timeDuration = null; - if (!isNull) { + if (hasValue) { timeDuration = TimeDuration.deserialize(byteBuffer); } - isNull = ReadWriteIOUtils.readBool(byteBuffer); + hasValue = ReadWriteIOUtils.readBool(byteBuffer); Symbol helperColumn = null; - if (!isNull) { + if (hasValue) { helperColumn = Symbol.deserialize(byteBuffer); } PlanNodeId planNodeId = PlanNodeId.deserialize(byteBuffer); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java index da83b7731247..d28901690e16 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java @@ -22,9 +22,13 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; + +import java.util.Collections; import static org.apache.iotdb.db.utils.constant.TestConstant.TIMESTAMP_STR; @@ -62,23 +66,29 @@ public PlanNode visitPlan(PlanNode node, Context context) { public PlanNode visitSort(SortNode node, Context context) { Context newContext = new Context(); PlanNode child = node.getChild().accept(this, newContext); + context.setHasSeenFill(newContext.hasSeenFill); OrderingScheme orderingScheme = node.getOrderingScheme(); - if (newContext.getTotalDeviceEntrySize() == 1 + if (!context.hasSeenFill() + && newContext.getTotalDeviceEntrySize() == 1 && TIMESTAMP_STR.equalsIgnoreCase(orderingScheme.getOrderBy().get(0).getName())) { return child; } - - return node.isOrderByAllIdsAndTime() ? child : node; + return !context.hasSeenFill() && node.isOrderByAllIdsAndTime() + ? child + : node.replaceChildren(Collections.singletonList(child)); } @Override public PlanNode visitStreamSort(StreamSortNode node, Context context) { - PlanNode child = node.getChild().accept(this, context); - return node.isOrderByAllIdsAndTime() - || node.getStreamCompareKeyEndIndex() - == node.getOrderingScheme().getOrderBy().size() - 1 + Context newContext = new Context(); + PlanNode child = node.getChild().accept(this, newContext); + context.setHasSeenFill(newContext.hasSeenFill); + return !context.hasSeenFill() + && (node.isOrderByAllIdsAndTime() + || node.getStreamCompareKeyEndIndex() + == node.getOrderingScheme().getOrderBy().size() - 1) ? child - : node; + : node.replaceChildren(Collections.singletonList(child)); } @Override @@ -86,11 +96,23 @@ public PlanNode visitTableScan(TableScanNode node, Context context) { context.addDeviceEntrySize(node.getDeviceEntries().size()); return node; } + + @Override + public PlanNode visitFill(FillNode node, Context context) { + PlanNode newNode = node.clone(); + for (PlanNode child : node.getChildren()) { + newNode.addChild(child.accept(this, context)); + } + context.setHasSeenFill(!(node instanceof ValueFillNode)); + return newNode; + } } private static class Context { private int totalDeviceEntrySize = 0; + private boolean hasSeenFill = false; + Context() {} public void addDeviceEntrySize(int deviceEntrySize) { @@ -100,5 +122,13 @@ public void addDeviceEntrySize(int deviceEntrySize) { public int getTotalDeviceEntrySize() { return totalDeviceEntrySize; } + + public boolean hasSeenFill() { + return hasSeenFill; + } + + public void setHasSeenFill(boolean hasSeenFill) { + this.hasSeenFill = hasSeenFill; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformSortToStreamSort.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformSortToStreamSort.java index 16bc3b4eeba4..c9b3cc071d0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformSortToStreamSort.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/TransformSortToStreamSort.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.FillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; @@ -135,17 +136,28 @@ private boolean isOrderByAllIdsAndTime( orderingScheme.getOrderBy().get(streamSortIndex + 1).getName()); } + @Override + public PlanNode visitFill(FillNode node, Context context) { + PlanNode newNode = node.clone(); + for (PlanNode child : node.getChildren()) { + newNode.addChild(child.accept(this, context)); + } + return newNode; + } + @Override public PlanNode visitTableScan(TableScanNode node, Context context) { context.setTableScanNode(node); return node; } + @Override public PlanNode visitAggregation(AggregationNode node, Context context) { context.setCanTransform(false); return visitSingleChildProcess(node, context); } + @Override public PlanNode visitAggregationTableScan(AggregationTableScanNode node, Context context) { context.setCanTransform(false); return visitTableScan(node, context); @@ -154,6 +166,7 @@ public PlanNode visitAggregationTableScan(AggregationTableScanNode node, Context private static class Context { private TableScanNode tableScanNode; + private boolean canTransform = true; public TableScanNode getTableScanNode() { @@ -171,5 +184,9 @@ public boolean canTransform() { public void setCanTransform(boolean canTransform) { this.canTransform = canTransform; } + + public boolean isCanTransform() { + return canTransform; + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index 73e59ec1cb79..1aba94853c85 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -994,7 +994,7 @@ public Node visitQueryNoWith(RelationalSqlParser.QueryNoWithContext ctx) { Optional fill = Optional.empty(); if (ctx.fillClause() != null) { - fill = visitIfPresent(ctx.fillClause(), Fill.class); + fill = visitIfPresent(ctx.fillClause().fillMethod(), Fill.class); } Optional orderBy = Optional.empty(); @@ -1061,12 +1061,13 @@ public Node visitPreviousFill(RelationalSqlParser.PreviousFillContext ctx) { } if (ctx.INTEGER_VALUE() != null) { - helperColumnIndex = new LongLiteral(getLocation(ctx.INTEGER_VALUE()), ctx.getText()); + helperColumnIndex = + new LongLiteral(getLocation(ctx.INTEGER_VALUE()), ctx.INTEGER_VALUE().getText()); } } else { if (ctx.INTEGER_VALUE() != null) { throw new SemanticException( - "Don't need to specify helper column index while timeDuration parameter is not specified."); + "Don't need to specify helper column index while timeDuration parameter is not specified"); } } return new Fill(getLocation(ctx), timeDuration, helperColumnIndex); @@ -1076,7 +1077,8 @@ public Node visitPreviousFill(RelationalSqlParser.PreviousFillContext ctx) { public Node visitLinearFill(RelationalSqlParser.LinearFillContext ctx) { if (ctx.INTEGER_VALUE() != null) { return new Fill( - getLocation(ctx), new LongLiteral(getLocation(ctx.INTEGER_VALUE()), ctx.getText())); + getLocation(ctx), + new LongLiteral(getLocation(ctx.INTEGER_VALUE()), ctx.INTEGER_VALUE().getText())); } else { return new Fill(getLocation(ctx)); } diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 995847244174..464f21a0fcc2 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -845,7 +845,7 @@ nonReserved | CACHE | CALL | CALLED | CASCADE | CATALOG | CATALOGS | CHAR | CHARACTER | CHARSET | CLEAR | CLUSTER | CLUSTERID | COLUMN | COLUMNS | COMMENT | COMMIT | COMMITTED | CONDITION | CONDITIONAL | CONFIGNODES | CONFIGURATION | CONNECTOR | COPARTITION | COUNT | CURRENT | DATA | DATABASE | DATABASES | DATANODES | DATE | DAY | DECLARE | DEFAULT | DEFINE | DEFINER | DENY | DESC | DESCRIPTOR | DETAILS| DETERMINISTIC | DEVICES | DISTRIBUTED | DO | DOUBLE | ELSEIF | EMPTY | ENCODING | ERROR | EXCLUDING | EXPLAIN | EXTRACTOR - | FETCH | FILL | FILTER | FINAL | FIRST | FLUSH | FOLLOWING | FORMAT | FUNCTION | FUNCTIONS + | FETCH | FILTER | FINAL | FIRST | FLUSH | FOLLOWING | FORMAT | FUNCTION | FUNCTIONS | GRACE | GRANT | GRANTED | GRANTS | GRAPHVIZ | GROUPS | HOUR | ID | INDEX | INDEXES | IF | IGNORE | IMMEDIATE | INCLUDING | INITIAL | INPUT | INTERVAL | INVOKER | IO | ITERATE | ISOLATION