CD backports (#1318)

* CD backports follow huggingface/safetensors#317 * fix node bindings? `cargo check` doesnt work on my local configuration from `tokenizers/bindings/node/native` i don't think it will be a problem but i have difficulty telling * backport #315 * safetensors#317 back ports
huggingface · Aug 10, 2023 · 862046a · 862046a
1 parent d47d3e3
commit 862046a
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 9 deletions.
diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml
@@ -73,13 +73,13 @@ jobs:
 
 
   create_wheels_others_64bit:
-    name: Create wheels for other OSes
+    name: Other OSes
     env:
       MACOSX_DEPLOYMENT_TARGET: 10.11
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [windows-latest, macos-10.15]
+        os: [windows-latest, macos-latest]
         python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
     steps:
       - name: Checkout repository
@@ -109,7 +109,49 @@ jobs:
       - name: Rename wheels
         shell: bash
         working-directory: ./bindings/python/dist
-        run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} ; done
+        run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done
+
+      - name: Upload wheels
+        shell: bash
+        run: |
+          pip install awscli
+          aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"
+
+  create_wheels_macos_13:
+    name: MacOS 13
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-13]
+        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+
+      - name: Install Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+
+      - name: Install dependencies
+        run: |
+          # On old versions of python there is an old version of setuptools already installed
+          pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall
+      - name: Build wheel
+        working-directory: ./bindings/python
+        run: python setup.py bdist_wheel
+
+      - name: Rename wheels
+        shell: bash
+        working-directory: ./bindings/python/dist
+        run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done
 
       - name: Upload wheels
         shell: bash
@@ -122,10 +164,11 @@ jobs:
     runs-on: macos-arm64
     strategy:
       matrix:
-        python: ["3.8.16", "3.9.13", "3.10.6", "3.11.1"]
+        python: [ "3.9.13", "3.10.6", "3.11.1"]
+        target: ["12.0", "13.0"]
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Install Rust
         uses: actions-rs/toolchain@v1
@@ -145,17 +188,17 @@ jobs:
           which python
           pip install -U setuptools wheel setuptools-rust awscli
           cd ./bindings/python
-          python setup.py bdist_wheel
+          MACOSX_DEPLOYMENT_TARGET=${{ matrix.target }} python setup.py bdist_wheel
           cd ../../
           aws s3 sync --exact-timestamps ./bindings/python/dist "s3://tokenizers-releases/python/$DIST_DIR"
 
   upload_package:
     name: Upload package to PyPi
     runs-on: ubuntu-latest
-    needs: [create_wheels_manylinux, create_wheels_windows_32bit, create_wheels_others_64bit, create_wheels_macos_arm64]
+    needs: [create_wheels_manylinux, create_wheels_windows_32bit, create_wheels_others_64bit, create_wheels_macos_arm64, create_wheels_macos_13]
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
 
       - name: Install Python
         uses: actions/setup-python@v1

diff --git a/bindings/node/native/src/tokenizer.rs b/bindings/node/native/src/tokenizer.rs
@@ -701,7 +701,7 @@ declare_types! {
             let params_obj = neon_serde::to_value(&mut cx, &TruncationParams(options.clone()))?;
             let mut this = cx.this();
             let guard = cx.lock();
-            this.borrow_mut(&guard)
+            let _ = this.borrow_mut(&guard)
                 .tokenizer.write().unwrap()
                 .with_truncation(Some(options));