feat(ci): support building python on windows

.bazelversion

@@ -0,0 +1 @@
+6.3.2

.github/workflows/ci.yml

@@ -185,7 +185,7 @@
     name: Rust CI
     strategy:
       matrix:
         os: [ubuntu-latest, macos-13, macos-14] # macos-13: x86, macos-14: arm64
     runs-on: ${{ matrix.os }}
     timeout-minutes: 45
     steps:
@@ -201,7 +201,7 @@
     name: C++ CI
     strategy:
       matrix:
         os: [ubuntu-latest, macos-13, macos-14, windows-2022] # macos-13: x86, macos-14: arm64
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
@@ -215,19 +215,27 @@
     name: Python CI
     # Fix python 3.6 install issue, see
     # https://github.com/rwth-i6/returnn/commit/38ecab17d781c4b74db6a174c8097187380b4ddc
-    runs-on: ubuntu-20.04
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
         python-version: [3.8, 3.12]
+        os: [ubuntu-20.04, windows-2022]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install bazel
-        run: ./ci/run_ci.sh install_bazel
+        shell: bash
+        run: |
+             if [ "$RUNNER_OS" == "Windows" ]; then
+                 ./ci/run_ci.sh install_bazel_windows
+             else
+                 ./ci/run_ci.sh install_bazel
+             fi
       - name: Run Python CI
+        shell: bash
         run: ./ci/run_ci.sh python
 
   go:

.gitignore

@@ -1,6 +1,7 @@
 **/*.cpp
 **/*.so
 **/*.dylib
+**/*.pyd
 bazel-*
 .whl
 python/.cache

BUILD

@@ -103,10 +103,19 @@ genrule(
         set -e
         set -x
         WORK_DIR=$$(pwd)
-        cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury"
-        cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3"
-        cp -f $(location python/pyfury/format/_format.so) "$$WORK_DIR/python/pyfury/format"
-        cp -f $(location python/pyfury/_serialization.so) "$$WORK_DIR/python/pyfury"
+        u_name=`uname -s`
+        if [ "$${u_name: 0: 4}" == "MING" ] || [ "$${u_name: 0: 4}" == "MSYS" ]
+        then
+            cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury/_util.pyd"
+            cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3/mmh3.pyd"
+            cp -f $(location python/pyfury/format/_format.so) "$$WORK_DIR/python/pyfury/format/_format.pyd"
+            cp -f $(location python/pyfury/_serialization.so) "$$WORK_DIR/python/pyfury/_serialization.pyd"
+        else
+            cp -f $(location python/pyfury/_util.so) "$$WORK_DIR/python/pyfury"
+            cp -f $(location python/pyfury/lib/mmh3/mmh3.so) "$$WORK_DIR/python/pyfury/lib/mmh3"
+            cp -f $(location python/pyfury/format/_format.so) "$$WORK_DIR/python/pyfury/format"
+            cp -f $(location python/pyfury/_serialization.so) "$$WORK_DIR/python/pyfury"
+        fi
         echo $$(date) > $@
     """,
     local = 1,

ci/deploy.sh

@@ -47,9 +47,8 @@ VERSIONS=("3.7"
           "3.11"
           "3.12")
 
-source $(conda info --base)/etc/profile.d/conda.sh
-
 create_py_envs() {
+  source $(conda info --base)/etc/profile.d/conda.sh
   for version in "${VERSIONS[@]}"; do
     conda create -y --name "py$version" python="$version"
   done
@@ -94,6 +93,7 @@ deploy_jars() {
 }
 
 deploy_python() {
+  source $(conda info --base)/etc/profile.d/conda.sh
   if command -v pyenv; then
     pyenv local system
   fi

ci/run_ci.sh

@@ -78,6 +78,12 @@ install_bazel() {
   fi
 }
 
+install_bazel_windows() {
+  choco install bazel --version=6.3.2 --force
+  VERSION=`bazel version`
+  echo "bazel version: $VERSION"
+}
+
 JDKS=(
 "zulu21.28.85-ca-jdk21.0.0-linux_x64"
 "zulu17.44.17-ca-crac-jdk17.0.8-linux_x64"

cpp/fury/thirdparty/MurmurHash3.cc

@@ -19,6 +19,7 @@
 #define FORCE_INLINE __forceinline
 
 #include <cstdint>
+#include <cstdlib>
 
 #define ROTL32(x, y) _rotl(x, y)
 #define ROTL64(x, y) _rotl64(x, y)

cpp/fury/util/logging.cc

@@ -52,7 +52,7 @@ std::string GetCallTrace() {
 
 std::unordered_map<FuryLogLevel, std::string> log_level_to_str = {
     {FuryLogLevel::DEBUG, "DEBUG"},     {FuryLogLevel::INFO, "INFO"},
-    {FuryLogLevel::WARNING, "WARNING"}, {FuryLogLevel::ERROR, "ERROR"},
+    {FuryLogLevel::WARNING, "WARNING"}, {FuryLogLevel::ERR, "ERROR"},
     {FuryLogLevel::FATAL, "FATAL"},
 };
 
@@ -77,7 +77,7 @@ FuryLogLevel FuryLog::GetLogLevel() {
     } else if (data == "warning") {
       severity_threshold = FuryLogLevel::WARNING;
     } else if (data == "error") {
-      severity_threshold = FuryLogLevel::ERROR;
+      severity_threshold = FuryLogLevel::ERR;
     } else if (data == "fatal") {
       severity_threshold = FuryLogLevel::FATAL;
     } else {

cpp/fury/util/logging.h

@@ -35,7 +35,7 @@ enum class FuryLogLevel {
   DEBUG = -1,
   INFO = 0,
   WARNING = 1,
-  ERROR = 2,
+  ERR = 2,
   FATAL = 3
 };
 

python/pyfury/_util.pyx

@@ -29,10 +29,13 @@ from libcpp cimport bool as c_bool
 from pyfury.includes.libutil cimport(
     CBuffer, AllocateBuffer, GetBit, SetBit, ClearBit, SetBitTo, CStatus, StatusCode, utf16HasSurrogatePairs
 )
+import os
 
 cdef int32_t max_buffer_size = 2 ** 31 - 1
 cdef int UTF16_LE = -1
 
+cdef c_bool _WINDOWS = os.name == 'nt'
+
 
 @cython.final
 cdef class Buffer:
@@ -686,6 +689,13 @@ cdef inline uint8_t* get_address(v):
         signed_int_data = v
         ptr = <uint8_t*>(&signed_int_data[0])
     elif dtype == "l":
+        if _WINDOWS:
+            signed_int_data = v
+            ptr = <uint8_t*>(&signed_int_data[0])
+        else:
+            signed_long_data = v
+            ptr = <uint8_t*>(&signed_long_data[0])
+    elif dtype == "q":
         signed_long_data = v
         ptr = <uint8_t*>(&signed_long_data[0])
     elif dtype == "f":

python/pyfury/serializer.py

@@ -43,6 +43,8 @@
     BufferObject,
 )
 
+_WINDOWS = os.name == "nt"
+
 from pyfury._serialization import ENABLE_FURY_CYTHON_SERIALIZATION
 
 if ENABLE_FURY_CYTHON_SERIALIZATION:
@@ -420,33 +422,63 @@ def xread(self, buffer):
 
 
 # Use numpy array or python array module.
-typecode_dict = {
-    # use bytes serializer for byte array.
-    "h": (2, Int16ArrayType, TypeId.INT16_ARRAY),
-    "i": (4, Int32ArrayType, TypeId.INT32_ARRAY),
-    "l": (8, Int64ArrayType, TypeId.INT64_ARRAY),
-    "f": (4, Float32ArrayType, TypeId.FLOAT32_ARRAY),
-    "d": (8, Float64ArrayType, TypeId.FLOAT64_ARRAY),
-}
-
-typeid_code = {
-    TypeId.INT16_ARRAY: "h",
-    TypeId.INT32_ARRAY: "i",
-    TypeId.INT64_ARRAY: "l",
-    TypeId.FLOAT32_ARRAY: "f",
-    TypeId.FLOAT64_ARRAY: "d",
-}
+typecode_dict = (
+    {
+        # use bytes serializer for byte array.
+        "h": (2, Int16ArrayType, TypeId.INT16_ARRAY),
+        "i": (4, Int32ArrayType, TypeId.INT32_ARRAY),
+        "l": (8, Int64ArrayType, TypeId.INT64_ARRAY),
+        "f": (4, Float32ArrayType, TypeId.FLOAT32_ARRAY),
+        "d": (8, Float64ArrayType, TypeId.FLOAT64_ARRAY),
+    }
+    if not _WINDOWS
+    else {
+        "h": (2, Int16ArrayType, TypeId.INT16_ARRAY),
+        "l": (4, Int32ArrayType, TypeId.INT32_ARRAY),
+        "q": (8, Int64ArrayType, TypeId.INT64_ARRAY),
+        "f": (4, Float32ArrayType, TypeId.FLOAT32_ARRAY),
+        "d": (8, Float64ArrayType, TypeId.FLOAT64_ARRAY),
+    }
+)
+
+typeid_code = (
+    {
+        TypeId.INT16_ARRAY: "h",
+        TypeId.INT32_ARRAY: "i",
+        TypeId.INT64_ARRAY: "l",
+        TypeId.FLOAT32_ARRAY: "f",
+        TypeId.FLOAT64_ARRAY: "d",
+    }
+    if not _WINDOWS
+    else {
+        TypeId.INT16_ARRAY: "h",
+        TypeId.INT32_ARRAY: "l",
+        TypeId.INT64_ARRAY: "q",
+        TypeId.FLOAT32_ARRAY: "f",
+        TypeId.FLOAT64_ARRAY: "d",
+    }
+)
 
 
 class PyArraySerializer(CrossLanguageCompatibleSerializer):
     typecode_dict = typecode_dict
-    typecodearray_type = {
-        "h": Int16ArrayType,
-        "i": Int32ArrayType,
-        "l": Int64ArrayType,
-        "f": Float32ArrayType,
-        "d": Float64ArrayType,
-    }
+    typecodearray_type = (
+        {
+            "h": Int16ArrayType,
+            "i": Int32ArrayType,
+            "l": Int64ArrayType,
+            "f": Float32ArrayType,
+            "d": Float64ArrayType,
+        }
+        if not _WINDOWS
+        else {
+            "h": Int16ArrayType,
+            "l": Int32ArrayType,
+            "q": Int64ArrayType,
+            "f": Float32ArrayType,
+            "d": Float64ArrayType,
+        }
+    )
 
     def __init__(self, fury, ftype, type_id: str):
         super().__init__(fury, ftype)
@@ -511,15 +543,26 @@ def read(self, buffer):
 
 
 if np:
-    _np_dtypes_dict = {
-        # use bytes serializer for byte array.
-        np.dtype(np.bool_): (1, "?", BoolNDArrayType, TypeId.BOOL_ARRAY),
-        np.dtype(np.int16): (2, "h", Int16NDArrayType, TypeId.INT16_ARRAY),
-        np.dtype(np.int32): (4, "i", Int32NDArrayType, TypeId.INT32_ARRAY),
-        np.dtype(np.int64): (8, "l", Int64NDArrayType, TypeId.INT64_ARRAY),
-        np.dtype(np.float32): (4, "f", Float32NDArrayType, TypeId.FLOAT32_ARRAY),
-        np.dtype(np.float64): (8, "d", Float64NDArrayType, TypeId.FLOAT64_ARRAY),
-    }
+    _np_dtypes_dict = (
+        {
+            # use bytes serializer for byte array.
+            np.dtype(np.bool_): (1, "?", BoolNDArrayType, TypeId.BOOL_ARRAY),
+            np.dtype(np.int16): (2, "h", Int16NDArrayType, TypeId.INT16_ARRAY),
+            np.dtype(np.int32): (4, "i", Int32NDArrayType, TypeId.INT32_ARRAY),
+            np.dtype(np.int64): (8, "l", Int64NDArrayType, TypeId.INT64_ARRAY),
+            np.dtype(np.float32): (4, "f", Float32NDArrayType, TypeId.FLOAT32_ARRAY),
+            np.dtype(np.float64): (8, "d", Float64NDArrayType, TypeId.FLOAT64_ARRAY),
+        }
+        if not _WINDOWS
+        else {
+            np.dtype(np.bool_): (1, "?", BoolNDArrayType, TypeId.BOOL_ARRAY),
+            np.dtype(np.int16): (2, "h", Int16NDArrayType, TypeId.INT16_ARRAY),
+            np.dtype(np.int32): (4, "l", Int32NDArrayType, TypeId.INT32_ARRAY),
+            np.dtype(np.int64): (8, "q", Int64NDArrayType, TypeId.INT64_ARRAY),
+            np.dtype(np.float32): (4, "f", Float32NDArrayType, TypeId.FLOAT32_ARRAY),
+            np.dtype(np.float64): (8, "d", Float64NDArrayType, TypeId.FLOAT64_ARRAY),
+        }
+    )
 else:
     _np_dtypes_dict = {}
 

python/pyfury/tests/test_serializer.py

@@ -19,6 +19,7 @@
 import datetime
 import gc
 import io
+import os
 import pickle
 import weakref
 from enum import Enum
@@ -249,6 +250,28 @@ def test_array_serializer(language):
         np.testing.assert_array_equal(new_arr, arr)
 
 
+def test_numpy_array_memoryview():
+    _WINDOWS = os.name == "nt"
+    if _WINDOWS:
+        arr = np.array(list(range(10)), dtype="int32")
+        view = memoryview(arr)
+        assert view.format == "l"
+        assert view.itemsize == 4
+        arr = np.array(list(range(10)), dtype="int64")
+        view = memoryview(arr)
+        assert view.format == "q"
+        assert view.itemsize == 8
+    else:
+        arr = np.array(list(range(10)), dtype="int32")
+        view = memoryview(arr)
+        assert view.format == "i"
+        assert view.itemsize == 4
+        arr = np.array(list(range(10)), dtype="int64")
+        view = memoryview(arr)
+        assert view.format == "l"
+        assert view.itemsize == 8
+
+
 def ser_de(fury, obj):
     binary = fury.serialize(obj)
     return fury.deserialize(binary)