[Py-Client] support new data types (#13494)

* [Py-Client] support new data types

* add test

* recover useless change

* more change

iotdb-client/client-py/SessionExample.py

@@ -18,7 +18,7 @@
 
 # Uncomment the following line to use apache-iotdb module installed by pip3
 import numpy as np
-
+from datetime import date
 from iotdb.Session import Session
 from iotdb.utils.BitMap import BitMap
 from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor
@@ -360,6 +360,57 @@
     while session_data_set.has_next():
         print(session_data_set.next())
 
+# insert tablet with new data types
+measurements_new_type = ["s_01", "s_02", "s_03", "s_04"]
+data_types_new_type = [
+    TSDataType.DATE,
+    TSDataType.TIMESTAMP,
+    TSDataType.BLOB,
+    TSDataType.STRING,
+]
+values_new_type = [
+    [date(2024, 1, 1), 1, b"\x12\x34", "test01"],
+    [date(2024, 1, 2), 2, b"\x12\x34", "test02"],
+    [date(2024, 1, 3), 3, b"\x12\x34", "test03"],
+    [date(2024, 1, 4), 4, b"\x12\x34", "test04"],
+]
+timestamps_new_type = [1, 2, 3, 4]
+tablet_new_type = Tablet(
+    "root.sg_test_01.d_04",
+    measurements_new_type,
+    data_types_new_type,
+    values_new_type,
+    timestamps_new_type,
+)
+session.insert_tablet(tablet_new_type)
+np_values_new_type = [
+    np.array([date(2024, 2, 4), date(2024, 3, 4), date(2024, 4, 4), date(2024, 5, 4)]),
+    np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype()),
+    np.array([b"\x12\x34", b"\x12\x34", b"\x12\x34", b"\x12\x34"]),
+    np.array(["test01", "test02", "test03", "test04"]),
+]
+np_timestamps_new_type = np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype())
+np_tablet_new_type = NumpyTablet(
+    "root.sg_test_01.d_04",
+    measurements_new_type,
+    data_types_new_type,
+    np_values_new_type,
+    np_timestamps_new_type,
+)
+session.insert_tablet(np_tablet_new_type)
+with session.execute_query_statement(
+    "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04"
+) as dataset:
+    print(dataset.get_column_names())
+    while dataset.has_next():
+        print(dataset.next())
+
+with session.execute_query_statement(
+    "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04"
+) as dataset:
+    df = dataset.todf()
+    print(df.to_string())
+
 # delete database
 session.delete_storage_group("root.sg_test_01")
 

iotdb-client/client-py/iotdb/Session.py

@@ -58,6 +58,7 @@
     TSLastDataQueryReq,
     TSInsertStringRecordsOfOneDeviceReq,
 )
+from .tsfile.utils.DateUtils import parse_date_to_int
 from .utils.IoTDBConnectionException import IoTDBConnectionException
 
 logger = logging.getLogger("IoTDB")
@@ -1442,6 +1443,36 @@ def value_to_bytes(data_types, values):
                 values_tobe_packed.append(b"\x05")
                 values_tobe_packed.append(len(value_bytes))
                 values_tobe_packed.append(value_bytes)
+            # TIMESTAMP
+            elif data_type == 8:
+                format_str_list.append("cq")
+                values_tobe_packed.append(b"\x08")
+                values_tobe_packed.append(value)
+            # DATE
+            elif data_type == 9:
+                format_str_list.append("ci")
+                values_tobe_packed.append(b"\x09")
+                values_tobe_packed.append(parse_date_to_int(value))
+            # BLOB
+            elif data_type == 10:
+                format_str_list.append("ci")
+                format_str_list.append(str(len(value)))
+                format_str_list.append("s")
+                values_tobe_packed.append(b"\x0a")
+                values_tobe_packed.append(len(value))
+                values_tobe_packed.append(value)
+            # STRING
+            elif data_type == 11:
+                if isinstance(value, str):
+                    value_bytes = bytes(value, "utf-8")
+                else:
+                    value_bytes = value
+                format_str_list.append("ci")
+                format_str_list.append(str(len(value_bytes)))
+                format_str_list.append("s")
+                values_tobe_packed.append(b"\x0b")
+                values_tobe_packed.append(len(value_bytes))
+                values_tobe_packed.append(value_bytes)
             else:
                 raise RuntimeError("Unsupported data type:" + str(data_type))
         format_str = "".join(format_str_list)

iotdb-client/client-py/iotdb/tsfile/utils/DateUtils.py

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+from datetime import date
+
+
+class DateTimeParseException(Exception):
+    pass
+
+
+def parse_int_to_date(date_int: int) -> date:
+    try:
+        year = date_int // 10000
+        month = (date_int // 100) % 100
+        day = date_int % 100
+        return date(year, month, day)
+    except ValueError as e:
+        raise DateTimeParseException("Invalid date format.") from e
+
+
+def parse_date_to_int(local_date: date) -> int:
+    if local_date is None:
+        raise DateTimeParseException("Date expression is none or empty.")
+    if local_date.year < 1000:
+        raise DateTimeParseException("Year must be between 1000 and 9999.")
+    return local_date.year * 10000 + local_date.month * 100 + local_date.day

iotdb-client/client-py/iotdb/utils/Field.py

@@ -17,7 +17,8 @@
 #
 
 # for package
-from .IoTDBConstants import TSDataType
+from iotdb.utils.IoTDBConstants import TSDataType
+from iotdb.tsfile.utils.DateUtils import parse_int_to_date
 import numpy as np
 import pandas as pd
 
@@ -36,15 +37,22 @@ def copy(field):
         if output.get_data_type() is not None:
             if output.get_data_type() == TSDataType.BOOLEAN:
                 output.set_bool_value(field.get_bool_value())
-            elif output.get_data_type() == TSDataType.INT32:
+            elif (
+                output.get_data_type() == TSDataType.INT32
+                or output.get_data_type() == TSDataType.DATE
+            ):
                 output.set_int_value(field.get_int_value())
             elif output.get_data_type() == TSDataType.INT64:
                 output.set_long_value(field.get_long_value())
             elif output.get_data_type() == TSDataType.FLOAT:
                 output.set_float_value(field.get_float_value())
             elif output.get_data_type() == TSDataType.DOUBLE:
                 output.set_double_value(field.get_double_value())
-            elif output.get_data_type() == TSDataType.TEXT:
+            elif (
+                output.get_data_type() == TSDataType.TEXT
+                or output.get_data_type() == TSDataType.STRING
+                or output.get_data_type() == TSDataType.BLOB
+            ):
                 output.set_binary_value(field.get_binary_value())
             else:
                 raise Exception(
@@ -80,6 +88,7 @@ def get_int_value(self):
             raise Exception("Null Field Exception!")
         if (
             self.__data_type != TSDataType.INT32
+            and self.__data_type != TSDataType.DATE
             or self.value is None
             or self.value is pd.NA
         ):
@@ -142,11 +151,26 @@ def get_binary_value(self):
             return None
         return self.value
 
+    def get_date_value(self):
+        if self.__data_type is None:
+            raise Exception("Null Field Exception!")
+        if (
+            self.__data_type != TSDataType.DATE
+            or self.value is None
+            or self.value is pd.NA
+        ):
+            return None
+        return parse_int_to_date(self.value)
+
     def get_string_value(self):
         if self.__data_type is None or self.value is None or self.value is pd.NA:
             return "None"
-        elif self.__data_type == 5:
+        # TEXT, STRING
+        elif self.__data_type == 5 or self.__data_type == 11:
             return self.value.decode("utf-8")
+        # BLOB
+        elif self.__data_type == 10:
+            return str(hex(int.from_bytes(self.value, byteorder="big")))
         else:
             return str(self.get_object_value(self.__data_type))
 
@@ -163,13 +187,18 @@ def get_object_value(self, data_type):
             return bool(self.value)
         elif data_type == 1:
             return np.int32(self.value)
-        elif data_type == 2:
+        elif data_type == 2 or data_type == 8:
             return np.int64(self.value)
         elif data_type == 3:
             return np.float32(self.value)
         elif data_type == 4:
             return np.float64(self.value)
-        return self.value
+        elif data_type == 9:
+            return parse_int_to_date(self.value)
+        elif data_type == 5 or data_type == 10 or data_type == 11:
+            return self.value
+        else:
+            raise RuntimeError("Unsupported data type:" + str(data_type))
 
     @staticmethod
     def get_field(value, data_type):

iotdb-client/client-py/iotdb/utils/IoTDBConstants.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
+from datetime import date
 from enum import unique, IntEnum
 import numpy as np
 
@@ -27,6 +28,10 @@ class TSDataType(IntEnum):
     FLOAT = 3
     DOUBLE = 4
     TEXT = 5
+    TIMESTAMP = 8
+    DATE = 9
+    BLOB = 10
+    STRING = 11
 
     def np_dtype(self):
         return {
@@ -35,7 +40,11 @@ def np_dtype(self):
             TSDataType.DOUBLE: np.dtype(">f8"),
             TSDataType.INT32: np.dtype(">i4"),
             TSDataType.INT64: np.dtype(">i8"),
-            TSDataType.TEXT: np.dtype("str"),
+            TSDataType.TEXT: str,
+            TSDataType.TIMESTAMP: np.dtype(">i8"),
+            TSDataType.DATE: date,
+            TSDataType.BLOB: bytes,
+            TSDataType.STRING: str,
         }[self]