diff --git a/iotdb-client/client-py/SessionExample.py b/iotdb-client/client-py/SessionExample.py index c489fe7d03d1..ca610de9a0c2 100644 --- a/iotdb-client/client-py/SessionExample.py +++ b/iotdb-client/client-py/SessionExample.py @@ -18,7 +18,7 @@ # Uncomment the following line to use apache-iotdb module installed by pip3 import numpy as np - +from datetime import date from iotdb.Session import Session from iotdb.utils.BitMap import BitMap from iotdb.utils.IoTDBConstants import TSDataType, TSEncoding, Compressor @@ -360,6 +360,57 @@ while session_data_set.has_next(): print(session_data_set.next()) +# insert tablet with new data types +measurements_new_type = ["s_01", "s_02", "s_03", "s_04"] +data_types_new_type = [ + TSDataType.DATE, + TSDataType.TIMESTAMP, + TSDataType.BLOB, + TSDataType.STRING, +] +values_new_type = [ + [date(2024, 1, 1), 1, b"\x12\x34", "test01"], + [date(2024, 1, 2), 2, b"\x12\x34", "test02"], + [date(2024, 1, 3), 3, b"\x12\x34", "test03"], + [date(2024, 1, 4), 4, b"\x12\x34", "test04"], +] +timestamps_new_type = [1, 2, 3, 4] +tablet_new_type = Tablet( + "root.sg_test_01.d_04", + measurements_new_type, + data_types_new_type, + values_new_type, + timestamps_new_type, +) +session.insert_tablet(tablet_new_type) +np_values_new_type = [ + np.array([date(2024, 2, 4), date(2024, 3, 4), date(2024, 4, 4), date(2024, 5, 4)]), + np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype()), + np.array([b"\x12\x34", b"\x12\x34", b"\x12\x34", b"\x12\x34"]), + np.array(["test01", "test02", "test03", "test04"]), +] +np_timestamps_new_type = np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype()) +np_tablet_new_type = NumpyTablet( + "root.sg_test_01.d_04", + measurements_new_type, + data_types_new_type, + np_values_new_type, + np_timestamps_new_type, +) +session.insert_tablet(np_tablet_new_type) +with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" +) as dataset: + print(dataset.get_column_names()) + while dataset.has_next(): + print(dataset.next()) + +with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" +) as dataset: + df = dataset.todf() + print(df.to_string()) + # delete database session.delete_storage_group("root.sg_test_01") diff --git a/iotdb-client/client-py/iotdb/Session.py b/iotdb-client/client-py/iotdb/Session.py index 098faf0d5208..12545e6f7a27 100644 --- a/iotdb-client/client-py/iotdb/Session.py +++ b/iotdb-client/client-py/iotdb/Session.py @@ -58,6 +58,7 @@ TSLastDataQueryReq, TSInsertStringRecordsOfOneDeviceReq, ) +from .tsfile.utils.DateUtils import parse_date_to_int from .utils.IoTDBConnectionException import IoTDBConnectionException logger = logging.getLogger("IoTDB") @@ -118,7 +119,7 @@ def init_from_node_urls( fetch_size=DEFAULT_FETCH_SIZE, zone_id=DEFAULT_ZONE_ID, enable_redirection=True, - sql_dialect=None, + sql_dialect=SQL_DIALECT, database=None, ): if node_urls is None: @@ -1520,6 +1521,36 @@ def value_to_bytes(data_types, values): values_tobe_packed.append(b"\x05") values_tobe_packed.append(len(value_bytes)) values_tobe_packed.append(value_bytes) + # TIMESTAMP + elif data_type == 8: + format_str_list.append("cq") + values_tobe_packed.append(b"\x08") + values_tobe_packed.append(value) + # DATE + elif data_type == 9: + format_str_list.append("ci") + values_tobe_packed.append(b"\x09") + values_tobe_packed.append(parse_date_to_int(value)) + # BLOB + elif data_type == 10: + format_str_list.append("ci") + format_str_list.append(str(len(value))) + format_str_list.append("s") + values_tobe_packed.append(b"\x0a") + values_tobe_packed.append(len(value)) + values_tobe_packed.append(value) + # STRING + elif data_type == 11: + if isinstance(value, str): + value_bytes = bytes(value, "utf-8") + else: + value_bytes = value + format_str_list.append("ci") + format_str_list.append(str(len(value_bytes))) + format_str_list.append("s") + values_tobe_packed.append(b"\x0b") + values_tobe_packed.append(len(value_bytes)) + values_tobe_packed.append(value_bytes) else: raise RuntimeError("Unsupported data type:" + str(data_type)) format_str = "".join(format_str_list) diff --git a/iotdb-client/client-py/iotdb/tsfile/utils/DateUtils.py b/iotdb-client/client-py/iotdb/tsfile/utils/DateUtils.py new file mode 100644 index 000000000000..4cafedea6183 --- /dev/null +++ b/iotdb-client/client-py/iotdb/tsfile/utils/DateUtils.py @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +from datetime import date + + +class DateTimeParseException(Exception): + pass + + +def parse_int_to_date(date_int: int) -> date: + try: + year = date_int // 10000 + month = (date_int // 100) % 100 + day = date_int % 100 + return date(year, month, day) + except ValueError as e: + raise DateTimeParseException("Invalid date format.") from e + + +def parse_date_to_int(local_date: date) -> int: + if local_date is None: + raise DateTimeParseException("Date expression is none or empty.") + if local_date.year < 1000: + raise DateTimeParseException("Year must be between 1000 and 9999.") + return local_date.year * 10000 + local_date.month * 100 + local_date.day diff --git a/iotdb-client/client-py/iotdb/utils/Field.py b/iotdb-client/client-py/iotdb/utils/Field.py index 7b4b05eab724..fdc1396c8fb3 100644 --- a/iotdb-client/client-py/iotdb/utils/Field.py +++ b/iotdb-client/client-py/iotdb/utils/Field.py @@ -17,7 +17,8 @@ # # for package -from .IoTDBConstants import TSDataType +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.tsfile.utils.DateUtils import parse_int_to_date import numpy as np import pandas as pd @@ -36,7 +37,10 @@ def copy(field): if output.get_data_type() is not None: if output.get_data_type() == TSDataType.BOOLEAN: output.set_bool_value(field.get_bool_value()) - elif output.get_data_type() == TSDataType.INT32: + elif ( + output.get_data_type() == TSDataType.INT32 + or output.get_data_type() == TSDataType.DATE + ): output.set_int_value(field.get_int_value()) elif ( output.get_data_type() == TSDataType.INT64 @@ -50,6 +54,7 @@ def copy(field): elif ( output.get_data_type() == TSDataType.TEXT or output.get_data_type() == TSDataType.STRING + or output.get_data_type() == TSDataType.BLOB ): output.set_binary_value(field.get_binary_value()) else: @@ -86,6 +91,7 @@ def get_int_value(self): raise Exception("Null Field Exception!") if ( self.__data_type != TSDataType.INT32 + and self.__data_type != TSDataType.DATE or self.value is None or self.value is pd.NA ): @@ -151,11 +157,26 @@ def get_binary_value(self): return None return self.value + def get_date_value(self): + if self.__data_type is None: + raise Exception("Null Field Exception!") + if ( + self.__data_type != TSDataType.DATE + or self.value is None + or self.value is pd.NA + ): + return None + return parse_int_to_date(self.value) + def get_string_value(self): if self.__data_type is None or self.value is None or self.value is pd.NA: return "None" + # TEXT, STRING elif self.__data_type == 5 or self.__data_type == 11: return self.value.decode("utf-8") + # BLOB + elif self.__data_type == 10: + return str(hex(int.from_bytes(self.value, byteorder="big"))) else: return str(self.get_object_value(self.__data_type)) @@ -172,13 +193,18 @@ def get_object_value(self, data_type): return bool(self.value) elif data_type == 1: return np.int32(self.value) - elif data_type == 2: + elif data_type == 2 or data_type == 8: return np.int64(self.value) elif data_type == 3: return np.float32(self.value) elif data_type == 4: return np.float64(self.value) - return self.value + elif data_type == 9: + return parse_int_to_date(self.value) + elif data_type == 5 or data_type == 10 or data_type == 11: + return self.value + else: + raise RuntimeError("Unsupported data type:" + str(data_type)) @staticmethod def get_field(value, data_type): diff --git a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py index daae37d23ba3..4b9082b5353f 100644 --- a/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py +++ b/iotdb-client/client-py/iotdb/utils/IoTDBConstants.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # +from datetime import date from enum import unique, IntEnum import numpy as np @@ -39,9 +40,11 @@ def np_dtype(self): TSDataType.DOUBLE: np.dtype(">f8"), TSDataType.INT32: np.dtype(">i4"), TSDataType.INT64: np.dtype(">i8"), - TSDataType.TEXT: np.dtype("str"), + TSDataType.TEXT: str, TSDataType.TIMESTAMP: np.dtype(">i8"), - TSDataType.STRING: np.dtype("str"), + TSDataType.DATE: date, + TSDataType.BLOB: bytes, + TSDataType.STRING: str, }[self] diff --git a/iotdb-client/client-py/iotdb/utils/IoTDBRpcDataSet.py b/iotdb-client/client-py/iotdb/utils/IoTDBRpcDataSet.py index 2aa885125d8b..73d4f317de6d 100644 --- a/iotdb-client/client-py/iotdb/utils/IoTDBRpcDataSet.py +++ b/iotdb-client/client-py/iotdb/utils/IoTDBRpcDataSet.py @@ -24,6 +24,7 @@ import pandas as pd from thrift.transport import TTransport from iotdb.thrift.rpc.IClientRPCService import TSFetchResultsReq, TSCloseOperationReq +from iotdb.tsfile.utils.DateUtils import parse_int_to_date from iotdb.utils.IoTDBConstants import TSDataType logger = logging.getLogger("IoTDB") @@ -168,25 +169,31 @@ def construct_one_data_frame(self): data_type = self.column_type_deduplicated_list[location] value_buffer = self.__query_data_set.valueList[location] value_buffer_len = len(value_buffer) + # DOUBLE if data_type == 4: data_array = np.frombuffer( value_buffer, np.dtype(np.double).newbyteorder(">") ) + # FLOAT elif data_type == 3: data_array = np.frombuffer( value_buffer, np.dtype(np.float32).newbyteorder(">") ) + # BOOLEAN elif data_type == 0: data_array = np.frombuffer(value_buffer, np.dtype("?")) - elif data_type == 1: + # INT32, DATE + elif data_type == 1 or data_type == 9: data_array = np.frombuffer( value_buffer, np.dtype(np.int32).newbyteorder(">") ) + # INT64, TIMESTAMP elif data_type == 2 or data_type == 8: data_array = np.frombuffer( value_buffer, np.dtype(np.int64).newbyteorder(">") ) - elif data_type == 5 or data_type == 11: + # TEXT, STRING, BLOB + elif data_type == 5 or data_type == 11 or data_type == 10: j = 0 offset = 0 data_array = [] @@ -208,10 +215,15 @@ def construct_one_data_frame(self): data_array = data_array.byteswap().view( data_array.dtype.newbyteorder("<") ) - # self.__query_data_set.valueList[location] = None if len(data_array) < total_length: - # INT32 or INT64 or boolean - if data_type == 0 or data_type == 1 or data_type == 2: + # INT32, INT64, BOOLEAN, TIMESTAMP, DATE + if ( + data_type == 0 + or data_type == 1 + or data_type == 2 + or data_type == 8 + or data_type == 9 + ): tmp_array = np.full(total_length, np.nan, np.float32) else: tmp_array = np.full(total_length, None, dtype=object) @@ -223,10 +235,13 @@ def construct_one_data_frame(self): bit_mask = bit_mask[:total_length] tmp_array[bit_mask] = data_array - if data_type == 1: + # INT32, DATE + if data_type == 1 or data_type == 9: tmp_array = pd.Series(tmp_array, dtype="Int32") + # INT64, TIMESTAMP elif data_type == 2 or data_type == 8: tmp_array = pd.Series(tmp_array, dtype="Int64") + # BOOLEAN elif data_type == 0: tmp_array = pd.Series(tmp_array, dtype="boolean") data_array = tmp_array @@ -251,7 +266,7 @@ def _has_next_result_set(self): return True return False - def resultset_to_pandas(self): + def result_set_to_pandas(self): result = {} for column_name in self.__column_name_list: result[column_name] = [] @@ -283,24 +298,30 @@ def resultset_to_pandas(self): data_type = self.column_type_deduplicated_list[location] value_buffer = self.__query_data_set.valueList[location] value_buffer_len = len(value_buffer) + # DOUBLE if data_type == 4: data_array = np.frombuffer( value_buffer, np.dtype(np.double).newbyteorder(">") ) + # FLOAT elif data_type == 3: data_array = np.frombuffer( value_buffer, np.dtype(np.float32).newbyteorder(">") ) + # BOOLEAN elif data_type == 0: data_array = np.frombuffer(value_buffer, np.dtype("?")) + # INT32 elif data_type == 1: data_array = np.frombuffer( value_buffer, np.dtype(np.int32).newbyteorder(">") ) + # INT64, TIMESTAMP elif data_type == 2 or data_type == 8: data_array = np.frombuffer( value_buffer, np.dtype(np.int64).newbyteorder(">") ) + # TEXT, STRING elif data_type == 5 or data_type == 11: j = 0 offset = 0 @@ -317,7 +338,30 @@ def resultset_to_pandas(self): data_array.append(value) j += 1 offset += length - data_array = np.array(data_array, dtype=object) + data_array = pd.Series(data_array).astype(str) + # BLOB + elif data_type == 10: + j = 0 + offset = 0 + data_array = [] + while offset < value_buffer_len: + length = int.from_bytes( + value_buffer[offset : offset + 4], + byteorder="big", + signed=False, + ) + offset += 4 + value = value_buffer[offset : offset + length] + data_array.append(value) + j += 1 + offset += length + data_array = pd.Series(data_array) + # DATE + elif data_type == 9: + data_array = np.frombuffer( + value_buffer, np.dtype(np.int32).newbyteorder(">") + ) + data_array = pd.Series(data_array).apply(parse_int_to_date) else: raise RuntimeError("unsupported data type {}.".format(data_type)) if data_array.dtype.byteorder == ">": @@ -327,13 +371,26 @@ def resultset_to_pandas(self): self.__query_data_set.valueList[location] = None tmp_array = [] if len(data_array) < total_length: - if data_type == 1 or data_type == 2 or data_type == 8: - tmp_array = np.full(total_length, np.nan, np.float32) + # BOOLEAN, INT32, INT64, TIMESTAMP + if ( + data_type == 0 + or data_type == 1 + or data_type == 2 + or data_type == 8 + ): + tmp_array = np.full(total_length, np.nan, dtype=np.float32) + # FLOAT, DOUBLE elif data_type == 3 or data_type == 4: - tmp_array = np.full(total_length, np.nan, data_array.dtype) - elif data_type == 0: - tmp_array = np.full(total_length, np.nan, np.float32) - elif data_type == 5 or data_type == 11 or data_type == 10: + tmp_array = np.full( + total_length, np.nan, dtype=data_array.dtype + ) + # TEXT, STRING, BLOB, DATE + elif ( + data_type == 5 + or data_type == 11 + or data_type == 10 + or data_type == 9 + ): tmp_array = np.full(total_length, None, dtype=data_array.dtype) bitmap_buffer = self.__query_data_set.bitmapList[location] diff --git a/iotdb-client/client-py/iotdb/utils/NumpyTablet.py b/iotdb-client/client-py/iotdb/utils/NumpyTablet.py index f00b61de5196..4ce623c9db90 100644 --- a/iotdb-client/client-py/iotdb/utils/NumpyTablet.py +++ b/iotdb-client/client-py/iotdb/utils/NumpyTablet.py @@ -18,8 +18,11 @@ import struct +import numpy as np from numpy import ndarray from typing import List + +from iotdb.tsfile.utils.DateUtils import parse_date_to_int from iotdb.utils.IoTDBConstants import TSDataType from iotdb.utils.BitMap import BitMap from iotdb.utils.Tablet import ColumnType @@ -49,7 +52,7 @@ def __init__( 2, id:1, attr:1, 2.0 3, id:2, attr:2, 3.0 Notice: The tablet will be sorted at the initialization by timestamps - :param insert_target_name: Str, DeviceId if using tree-view interfaces or TableName when using table-view interfaces. + :param insert_target_name: Str, DeviceId if using tree model or TableName when using table model. :param column_names: Str List, names of columns :param data_types: TSDataType List, specify value types for columns :param values: ndarray List, one ndarray contains the value of one column @@ -128,12 +131,22 @@ def get_binary_values(self): bs_len = 0 bs_list = [] for data_type, value in zip(self.__data_types, self.__values): + # BOOLEAN, INT32, INT64, FLOAT, DOUBLE, TIMESTAMP + if ( + data_type == 0 + or data_type == 1 + or data_type == 2 + or data_type == 3 + or data_type == 4 + or data_type == 8 + ): + bs = value.tobytes() # TEXT, STRING, BLOB - if data_type == 5 or data_type == 11 or data_type == 10: + elif data_type == 5 or data_type == 11 or data_type == 10: format_str_list = [">"] values_tobe_packed = [] for str_list in value: - # Fot TEXT, it's same as the original solution + # For TEXT, it's same as the original solution if isinstance(str_list, str): value_bytes = bytes(str_list, "utf-8") else: @@ -145,11 +158,18 @@ def get_binary_values(self): values_tobe_packed.append(value_bytes) format_str = "".join(format_str_list) bs = struct.pack(format_str, *values_tobe_packed) - # Non-TEXT + # DATE + elif data_type == 9: + bs = ( + np.vectorize(parse_date_to_int)(value) + .astype(np.dtype(">i4")) + .tobytes() + ) else: - bs = value.tobytes() + raise RuntimeError("Unsupported data type:" + str(data_type)) bs_list.append(bs) bs_len += len(bs) + if self.bitmaps is not None: format_str_list = [">"] values_tobe_packed = [] diff --git a/iotdb-client/client-py/iotdb/utils/RowRecord.py b/iotdb-client/client-py/iotdb/utils/RowRecord.py index 16a88f1edf9b..91bcebf948b0 100644 --- a/iotdb-client/client-py/iotdb/utils/RowRecord.py +++ b/iotdb-client/client-py/iotdb/utils/RowRecord.py @@ -17,15 +17,11 @@ # # for package -from .Field import Field - -# for debug -# from IoTDBConstants import TSDataType -# from Field import Field +from iotdb.utils.Field import Field class RowRecord(object): - def __init__(self, timestamp, field_list=None): + def __init__(self, timestamp, field_list: list = None): self.__timestamp = timestamp self.__field_list = field_list diff --git a/iotdb-client/client-py/iotdb/utils/SessionDataSet.py b/iotdb-client/client-py/iotdb/utils/SessionDataSet.py index c69aca3ebbe4..f4d63035efc5 100644 --- a/iotdb-client/client-py/iotdb/utils/SessionDataSet.py +++ b/iotdb-client/client-py/iotdb/utils/SessionDataSet.py @@ -125,30 +125,32 @@ def close_operation_handle(self): self.iotdb_rpc_data_set.close() def todf(self) -> pd.DataFrame: - return resultset_to_pandas(self) + return result_set_to_pandas(self) -def resultset_to_pandas(result_set: SessionDataSet) -> pd.DataFrame: +def result_set_to_pandas(result_set: SessionDataSet) -> pd.DataFrame: """ Transforms a SessionDataSet from IoTDB to a Pandas Data Frame Each Field from IoTDB is a column in Pandas :param result_set: :return: """ - return result_set.iotdb_rpc_data_set.resultset_to_pandas() + return result_set.iotdb_rpc_data_set.result_set_to_pandas() def get_typed_point(field: Field, none_value=None): choices = { # In Case of Boolean, cast to 0 / 1 - TSDataType.BOOLEAN: lambda field: 1 if field.get_bool_value() else 0, - TSDataType.TEXT: lambda field: field.get_string_value(), - TSDataType.FLOAT: lambda field: field.get_float_value(), - TSDataType.INT32: lambda field: field.get_int_value(), - TSDataType.DOUBLE: lambda field: field.get_double_value(), - TSDataType.INT64: lambda field: field.get_long_value(), - TSDataType.TIMESTAMP: lambda field: field.get_long_value(), - TSDataType.STRING: lambda field: field.get_string_value(), + TSDataType.BOOLEAN: lambda f: 1 if f.get_bool_value() else 0, + TSDataType.TEXT: lambda f: f.get_string_value(), + TSDataType.FLOAT: lambda f: f.get_float_value(), + TSDataType.INT32: lambda f: f.get_int_value(), + TSDataType.DOUBLE: lambda f: f.get_double_value(), + TSDataType.INT64: lambda f: f.get_long_value(), + TSDataType.TIMESTAMP: lambda f: f.get_long_value(), + TSDataType.STRING: lambda f: f.get_string_value(), + TSDataType.DATE: lambda f: f.get_date_value(), + TSDataType.BLOB: lambda f: f.get_binary_value(), } result_next_type: TSDataType = field.get_data_type() diff --git a/iotdb-client/client-py/iotdb/utils/Tablet.py b/iotdb-client/client-py/iotdb/utils/Tablet.py index 0f086530347b..fbae5fe22935 100644 --- a/iotdb-client/client-py/iotdb/utils/Tablet.py +++ b/iotdb-client/client-py/iotdb/utils/Tablet.py @@ -18,7 +18,9 @@ import struct from enum import unique, IntEnum -from typing import List +from typing import List, Union + +from iotdb.tsfile.utils.DateUtils import parse_date_to_int from iotdb.utils.BitMap import BitMap from iotdb.utils.IoTDBConstants import TSDataType @@ -59,7 +61,7 @@ def __init__( 2, id:1, attr:1, 2.0 3, id:2, attr:2, 3.0 Notice: The tablet will be sorted at the initialization by timestamps - :param insert_target_name: Str, DeviceId if using tree-view interfaces or TableName when using table-view interfaces. + :param insert_target_name: Str, DeviceId if using tree model or TableName when using table model. :param column_names: Str List, names of columns :param data_types: TSDataType List, specify value types for columns :param values: 2-D List, the values of each row should be the outer list element @@ -126,13 +128,14 @@ def get_binary_timestamps(self): def get_binary_values(self): format_str_list = [">"] values_tobe_packed = [] - bitmaps = [] + bitmaps: List[Union[BitMap, None]] = [] has_none = False for i in range(self.__column_number): bitmap = None bitmaps.append(bitmap) - data_type_value = self.__data_types[i] - if data_type_value == 0: + data_type = self.__data_types[i] + # BOOLEAN + if data_type == 0: format_str_list.append(str(self.__row_number)) format_str_list.append("?") for j in range(self.__row_number): @@ -142,8 +145,8 @@ def get_binary_values(self): values_tobe_packed.append(False) self.__mark_none_value(bitmaps, i, j) has_none = True - - elif data_type_value == 1: + # INT32 + elif data_type == 1: format_str_list.append(str(self.__row_number)) format_str_list.append("i") for j in range(self.__row_number): @@ -153,8 +156,8 @@ def get_binary_values(self): values_tobe_packed.append(0) self.__mark_none_value(bitmaps, i, j) has_none = True - - elif data_type_value == 2: + # INT64 or TIMESTAMP + elif data_type == 2 or data_type == 8: format_str_list.append(str(self.__row_number)) format_str_list.append("q") for j in range(self.__row_number): @@ -164,8 +167,8 @@ def get_binary_values(self): values_tobe_packed.append(0) self.__mark_none_value(bitmaps, i, j) has_none = True - - elif data_type_value == 3: + # FLOAT + elif data_type == 3: format_str_list.append(str(self.__row_number)) format_str_list.append("f") for j in range(self.__row_number): @@ -175,8 +178,8 @@ def get_binary_values(self): values_tobe_packed.append(0) self.__mark_none_value(bitmaps, i, j) has_none = True - - elif data_type_value == 4: + # DOUBLE + elif data_type == 4: format_str_list.append(str(self.__row_number)) format_str_list.append("d") for j in range(self.__row_number): @@ -186,8 +189,8 @@ def get_binary_values(self): values_tobe_packed.append(0) self.__mark_none_value(bitmaps, i, j) has_none = True - - elif data_type_value == 5 or data_type_value == 11: + # TEXT, STRING, BLOB + elif data_type == 5 or data_type == 11 or data_type == 10: for j in range(self.__row_number): if self.__values[j][i] is not None: if isinstance(self.__values[j][i], str): @@ -208,7 +211,19 @@ def get_binary_values(self): values_tobe_packed.append(value_bytes) self.__mark_none_value(bitmaps, i, j) has_none = True - + # DATE + elif data_type == 9: + format_str_list.append(str(self.__row_number)) + format_str_list.append("i") + for j in range(self.__row_number): + if self.__values[j][i] is not None: + values_tobe_packed.append( + parse_date_to_int(self.__values[j][i]) + ) + else: + values_tobe_packed.append(0) + self.__mark_none_value(bitmaps, i, j) + has_none = True else: raise RuntimeError("Unsupported data type:" + str(self.__data_types[i])) diff --git a/iotdb-client/client-py/tests/integration/test_new_data_types.py b/iotdb-client/client-py/tests/integration/test_new_data_types.py new file mode 100644 index 000000000000..3452d6c1b7bd --- /dev/null +++ b/iotdb-client/client-py/tests/integration/test_new_data_types.py @@ -0,0 +1,157 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +from datetime import date + +import numpy as np + +from iotdb.Session import Session +from iotdb.SessionPool import PoolConfig, create_session_pool +from iotdb.utils.IoTDBConstants import TSDataType +from iotdb.utils.NumpyTablet import NumpyTablet +from iotdb.utils.Tablet import Tablet +from iotdb.IoTDBContainer import IoTDBContainer + + +def test_session(): + session_test() + + +def test_session_pool(): + session_test(True) + + +def session_test(use_session_pool=False): + with IoTDBContainer("iotdb:dev") as db: + db: IoTDBContainer + + if use_session_pool: + pool_config = PoolConfig( + db.get_container_host_ip(), + db.get_exposed_port(6667), + "root", + "root", + None, + 1024, + "Asia/Shanghai", + 3, + ) + session_pool = create_session_pool(pool_config, 1, 3000) + session = session_pool.get_session() + else: + session = Session(db.get_container_host_ip(), db.get_exposed_port(6667)) + session.open(False) + + if not session.is_open(): + print("can't open session") + exit(1) + + device_id = "root.sg_test_01.d_04" + measurements_new_type = ["s_01", "s_02", "s_03", "s_04"] + data_types_new_type = [ + TSDataType.DATE, + TSDataType.TIMESTAMP, + TSDataType.BLOB, + TSDataType.STRING, + ] + values_new_type = [ + [date(2024, 1, 1), 1, b"\x12\x34", "test01"], + [date(2024, 1, 2), 2, b"\x12\x34", "test02"], + [date(2024, 1, 3), 3, b"\x12\x34", "test03"], + [date(2024, 1, 4), 4, b"\x12\x34", "test04"], + ] + timestamps_new_type = [1, 2, 3, 4] + tablet_new_type = Tablet( + device_id, + measurements_new_type, + data_types_new_type, + values_new_type, + timestamps_new_type, + ) + session.insert_tablet(tablet_new_type) + np_values_new_type = [ + np.array( + [date(2024, 1, 5), date(2024, 1, 6), date(2024, 1, 7), date(2024, 1, 8)] + ), + np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype()), + np.array([b"\x12\x34", b"\x12\x34", b"\x12\x34", b"\x12\x34"]), + np.array(["test05", "test06", "test07", "test08"]), + ] + np_timestamps_new_type = np.array([5, 6, 7, 8], TSDataType.INT64.np_dtype()) + np_tablet_new_type = NumpyTablet( + device_id, + measurements_new_type, + data_types_new_type, + np_values_new_type, + np_timestamps_new_type, + ) + session.insert_tablet(np_tablet_new_type) + session.insert_records( + [device_id, device_id], + [9, 10], + [measurements_new_type, measurements_new_type], + [data_types_new_type, data_types_new_type], + [ + [date(2024, 1, 9), 9, b"\x12\x34", "test09"], + [date(2024, 1, 10), 10, b"\x12\x34", "test010"], + ], + ) + + with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" + ) as dataset: + print(dataset.get_column_names()) + while dataset.has_next(): + print(dataset.next()) + + with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" + ) as dataset: + df = dataset.todf() + print(df.to_string()) + + with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" + ) as dataset: + cnt = 0 + while dataset.has_next(): + row_record = dataset.next() + timestamp = row_record.get_timestamp() + assert row_record.get_fields()[0].get_date_value() == date( + 2024, 1, timestamp + ) + assert ( + row_record.get_fields()[1].get_object_value(TSDataType.TIMESTAMP) + == timestamp + ) + assert row_record.get_fields()[2].get_binary_value() == b"\x12\x34" + assert row_record.get_fields()[3].get_string_value() == "test0" + str( + timestamp + ) + cnt += 1 + assert cnt == 10 + + with session.execute_query_statement( + "select s_01,s_02,s_03,s_04 from root.sg_test_01.d_04" + ) as dataset: + df = dataset.todf() + rows, columns = df.shape + assert rows == 10 + assert columns == 5 + + # close session connection. + session.close()