diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 0000000000..f9da12e118 --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +6.3.2 \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72201cff97..0bb76ce1dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,10 +215,11 @@ jobs: name: Python CI # Fix python 3.6 install issue, see # https://github.com/rwth-i6/returnn/commit/38ecab17d781c4b74db6a174c8097187380b4ddc - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} strategy: matrix: python-version: [3.8, 3.12] + os: [ubuntu-20.04, windows-2022] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -226,8 +227,15 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install bazel - run: ./ci/run_ci.sh install_bazel + shell: bash + run: | + if [ "$RUNNER_OS" == "Windows" ]; then + ./ci/run_ci.sh install_bazel_windows + else + ./ci/run_ci.sh install_bazel + fi - name: Run Python CI + shell: bash run: ./ci/run_ci.sh python go: diff --git a/.gitignore b/.gitignore index 53cfc24de2..f4faab088a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ **/*.cpp **/*.so **/*.dylib +**/*.pyd bazel-* .whl python/.cache diff --git a/BUILD b/BUILD index d5f1063377..cc438bd3fb 100644 --- a/BUILD +++ b/BUILD @@ -103,10 +103,19 @@ genrule( set -e set -x WORK_DIR=$$(pwd) - cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury" - cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3" - cp -f $(location python/pyfury/format/_format.so) "$$WORK_DIR/python/pyfury/format" - cp -f $(location python/pyfury/_serialization.so) "$$WORK_DIR/python/pyfury" + u_name=`uname -s` + if [ "$${u_name: 0: 4}" == "MING" ] || [ "$${u_name: 0: 4}" == "MSYS" ] + then + cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury/_util.pyd" + cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3/mmh3.pyd" + cp -f $(location python/pyfury/format/_format.so) "$$WORK_DIR/python/pyfury/format/_format.pyd" + cp -f $(location python/pyfury/_serialization.so) "$$WORK_DIR/python/pyfury/_serialization.pyd" + else + cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury" + cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3" + cp -f $(location python/pyfury/format/_format.so) "$$WORK_DIR/python/pyfury/format" + cp -f $(location python/pyfury/_serialization.so) "$$WORK_DIR/python/pyfury" + fi echo $$(date) > $@ """, local = 1, diff --git a/ci/deploy.sh b/ci/deploy.sh index e514848c00..4a8ab181bd 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -47,9 +47,8 @@ VERSIONS=("3.7" "3.11" "3.12") -source $(conda info --base)/etc/profile.d/conda.sh - create_py_envs() { + source $(conda info --base)/etc/profile.d/conda.sh for version in "${VERSIONS[@]}"; do conda create -y --name "py$version" python="$version" done @@ -94,6 +93,7 @@ deploy_jars() { } deploy_python() { + source $(conda info --base)/etc/profile.d/conda.sh if command -v pyenv; then pyenv local system fi diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 14664e9bb2..e562044171 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -78,6 +78,12 @@ install_bazel() { fi } +install_bazel_windows() { + choco install bazel --version=6.3.2 --force + VERSION=`bazel version` + echo "bazel version: $VERSION" +} + JDKS=( "zulu21.28.85-ca-jdk21.0.0-linux_x64" "zulu17.44.17-ca-crac-jdk17.0.8-linux_x64" diff --git a/cpp/fury/thirdparty/MurmurHash3.cc b/cpp/fury/thirdparty/MurmurHash3.cc index c9d144692d..7267187cc1 100644 --- a/cpp/fury/thirdparty/MurmurHash3.cc +++ b/cpp/fury/thirdparty/MurmurHash3.cc @@ -19,6 +19,7 @@ #define FORCE_INLINE __forceinline #include +#include #define ROTL32(x, y) _rotl(x, y) #define ROTL64(x, y) _rotl64(x, y) diff --git a/cpp/fury/util/logging.cc b/cpp/fury/util/logging.cc index c52d5feb76..9718a31be3 100644 --- a/cpp/fury/util/logging.cc +++ b/cpp/fury/util/logging.cc @@ -52,7 +52,7 @@ std::string GetCallTrace() { std::unordered_map log_level_to_str = { {FuryLogLevel::DEBUG, "DEBUG"}, {FuryLogLevel::INFO, "INFO"}, - {FuryLogLevel::WARNING, "WARNING"}, {FuryLogLevel::ERROR, "ERROR"}, + {FuryLogLevel::WARNING, "WARNING"}, {FuryLogLevel::ERR, "ERROR"}, {FuryLogLevel::FATAL, "FATAL"}, }; @@ -77,7 +77,7 @@ FuryLogLevel FuryLog::GetLogLevel() { } else if (data == "warning") { severity_threshold = FuryLogLevel::WARNING; } else if (data == "error") { - severity_threshold = FuryLogLevel::ERROR; + severity_threshold = FuryLogLevel::ERR; } else if (data == "fatal") { severity_threshold = FuryLogLevel::FATAL; } else { diff --git a/cpp/fury/util/logging.h b/cpp/fury/util/logging.h index 1770e3feb5..addb21027a 100644 --- a/cpp/fury/util/logging.h +++ b/cpp/fury/util/logging.h @@ -35,7 +35,7 @@ enum class FuryLogLevel { DEBUG = -1, INFO = 0, WARNING = 1, - ERROR = 2, + ERR = 2, FATAL = 3 }; diff --git a/python/pyfury/_util.pyx b/python/pyfury/_util.pyx index 3d0ac05fd9..de4eab33c7 100644 --- a/python/pyfury/_util.pyx +++ b/python/pyfury/_util.pyx @@ -29,10 +29,13 @@ from libcpp cimport bool as c_bool from pyfury.includes.libutil cimport( CBuffer, AllocateBuffer, GetBit, SetBit, ClearBit, SetBitTo, CStatus, StatusCode, utf16HasSurrogatePairs ) +import os cdef int32_t max_buffer_size = 2 ** 31 - 1 cdef int UTF16_LE = -1 +cdef c_bool _WINDOWS = os.name == 'nt' + @cython.final cdef class Buffer: @@ -686,6 +689,13 @@ cdef inline uint8_t* get_address(v): signed_int_data = v ptr = (&signed_int_data[0]) elif dtype == "l": + if _WINDOWS: + signed_int_data = v + ptr = (&signed_int_data[0]) + else: + signed_long_data = v + ptr = (&signed_long_data[0]) + elif dtype == "q": signed_long_data = v ptr = (&signed_long_data[0]) elif dtype == "f": diff --git a/python/pyfury/serializer.py b/python/pyfury/serializer.py index f7276f7d88..b420423991 100644 --- a/python/pyfury/serializer.py +++ b/python/pyfury/serializer.py @@ -43,6 +43,8 @@ BufferObject, ) +_WINDOWS = os.name == "nt" + from pyfury._serialization import ENABLE_FURY_CYTHON_SERIALIZATION if ENABLE_FURY_CYTHON_SERIALIZATION: @@ -420,33 +422,63 @@ def xread(self, buffer): # Use numpy array or python array module. -typecode_dict = { - # use bytes serializer for byte array. - "h": (2, Int16ArrayType, TypeId.INT16_ARRAY), - "i": (4, Int32ArrayType, TypeId.INT32_ARRAY), - "l": (8, Int64ArrayType, TypeId.INT64_ARRAY), - "f": (4, Float32ArrayType, TypeId.FLOAT32_ARRAY), - "d": (8, Float64ArrayType, TypeId.FLOAT64_ARRAY), -} - -typeid_code = { - TypeId.INT16_ARRAY: "h", - TypeId.INT32_ARRAY: "i", - TypeId.INT64_ARRAY: "l", - TypeId.FLOAT32_ARRAY: "f", - TypeId.FLOAT64_ARRAY: "d", -} +typecode_dict = ( + { + # use bytes serializer for byte array. + "h": (2, Int16ArrayType, TypeId.INT16_ARRAY), + "i": (4, Int32ArrayType, TypeId.INT32_ARRAY), + "l": (8, Int64ArrayType, TypeId.INT64_ARRAY), + "f": (4, Float32ArrayType, TypeId.FLOAT32_ARRAY), + "d": (8, Float64ArrayType, TypeId.FLOAT64_ARRAY), + } + if not _WINDOWS + else { + "h": (2, Int16ArrayType, TypeId.INT16_ARRAY), + "l": (4, Int32ArrayType, TypeId.INT32_ARRAY), + "q": (8, Int64ArrayType, TypeId.INT64_ARRAY), + "f": (4, Float32ArrayType, TypeId.FLOAT32_ARRAY), + "d": (8, Float64ArrayType, TypeId.FLOAT64_ARRAY), + } +) + +typeid_code = ( + { + TypeId.INT16_ARRAY: "h", + TypeId.INT32_ARRAY: "i", + TypeId.INT64_ARRAY: "l", + TypeId.FLOAT32_ARRAY: "f", + TypeId.FLOAT64_ARRAY: "d", + } + if not _WINDOWS + else { + TypeId.INT16_ARRAY: "h", + TypeId.INT32_ARRAY: "l", + TypeId.INT64_ARRAY: "q", + TypeId.FLOAT32_ARRAY: "f", + TypeId.FLOAT64_ARRAY: "d", + } +) class PyArraySerializer(CrossLanguageCompatibleSerializer): typecode_dict = typecode_dict - typecodearray_type = { - "h": Int16ArrayType, - "i": Int32ArrayType, - "l": Int64ArrayType, - "f": Float32ArrayType, - "d": Float64ArrayType, - } + typecodearray_type = ( + { + "h": Int16ArrayType, + "i": Int32ArrayType, + "l": Int64ArrayType, + "f": Float32ArrayType, + "d": Float64ArrayType, + } + if not _WINDOWS + else { + "h": Int16ArrayType, + "l": Int32ArrayType, + "q": Int64ArrayType, + "f": Float32ArrayType, + "d": Float64ArrayType, + } + ) def __init__(self, fury, ftype, type_id: str): super().__init__(fury, ftype) @@ -511,15 +543,26 @@ def read(self, buffer): if np: - _np_dtypes_dict = { - # use bytes serializer for byte array. - np.dtype(np.bool_): (1, "?", BoolNDArrayType, TypeId.BOOL_ARRAY), - np.dtype(np.int16): (2, "h", Int16NDArrayType, TypeId.INT16_ARRAY), - np.dtype(np.int32): (4, "i", Int32NDArrayType, TypeId.INT32_ARRAY), - np.dtype(np.int64): (8, "l", Int64NDArrayType, TypeId.INT64_ARRAY), - np.dtype(np.float32): (4, "f", Float32NDArrayType, TypeId.FLOAT32_ARRAY), - np.dtype(np.float64): (8, "d", Float64NDArrayType, TypeId.FLOAT64_ARRAY), - } + _np_dtypes_dict = ( + { + # use bytes serializer for byte array. + np.dtype(np.bool_): (1, "?", BoolNDArrayType, TypeId.BOOL_ARRAY), + np.dtype(np.int16): (2, "h", Int16NDArrayType, TypeId.INT16_ARRAY), + np.dtype(np.int32): (4, "i", Int32NDArrayType, TypeId.INT32_ARRAY), + np.dtype(np.int64): (8, "l", Int64NDArrayType, TypeId.INT64_ARRAY), + np.dtype(np.float32): (4, "f", Float32NDArrayType, TypeId.FLOAT32_ARRAY), + np.dtype(np.float64): (8, "d", Float64NDArrayType, TypeId.FLOAT64_ARRAY), + } + if not _WINDOWS + else { + np.dtype(np.bool_): (1, "?", BoolNDArrayType, TypeId.BOOL_ARRAY), + np.dtype(np.int16): (2, "h", Int16NDArrayType, TypeId.INT16_ARRAY), + np.dtype(np.int32): (4, "l", Int32NDArrayType, TypeId.INT32_ARRAY), + np.dtype(np.int64): (8, "q", Int64NDArrayType, TypeId.INT64_ARRAY), + np.dtype(np.float32): (4, "f", Float32NDArrayType, TypeId.FLOAT32_ARRAY), + np.dtype(np.float64): (8, "d", Float64NDArrayType, TypeId.FLOAT64_ARRAY), + } + ) else: _np_dtypes_dict = {} diff --git a/python/pyfury/tests/test_serializer.py b/python/pyfury/tests/test_serializer.py index ec6d399d4a..9230580952 100644 --- a/python/pyfury/tests/test_serializer.py +++ b/python/pyfury/tests/test_serializer.py @@ -19,6 +19,7 @@ import datetime import gc import io +import os import pickle import weakref from enum import Enum @@ -249,6 +250,28 @@ def test_array_serializer(language): np.testing.assert_array_equal(new_arr, arr) +def test_numpy_array_memoryview(): + _WINDOWS = os.name == "nt" + if _WINDOWS: + arr = np.array(list(range(10)), dtype="int32") + view = memoryview(arr) + assert view.format == "l" + assert view.itemsize == 4 + arr = np.array(list(range(10)), dtype="int64") + view = memoryview(arr) + assert view.format == "q" + assert view.itemsize == 8 + else: + arr = np.array(list(range(10)), dtype="int32") + view = memoryview(arr) + assert view.format == "i" + assert view.itemsize == 4 + arr = np.array(list(range(10)), dtype="int64") + view = memoryview(arr) + assert view.format == "l" + assert view.itemsize == 8 + + def ser_de(fury, obj): binary = fury.serialize(obj) return fury.deserialize(binary)