Feat rye

bytepiece-py/.gitignore

@@ -1,72 +1,13 @@
-/target
-
-# Byte-compiled / optimized / DLL files
+# python generated files
 __pycache__/
-.pytest_cache/
-*.py[cod]
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-.venv/
-env/
-bin/
+*.py[oc]
 build/
-develop-eggs/
 dist/
-eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-include/
-man/
-venv/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-pip-selfcheck.json
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.cache
-nosetests.xml
-coverage.xml
-
-# Translations
-*.mo
-
-# Mr Developer
-.mr.developer.cfg
-.project
-.pydevproject
-
-# Rope
-.ropeproject
-
-# Django stuff:
-*.log
-*.pot
-
-.DS_Store
-
-# Sphinx documentation
-docs/_build/
-
-# PyCharm
-.idea/
-
-# VSCode
-.vscode/
-
-# Pyenv
-.python-version
+wheels/
+*.egg-info
+# Rust
+target/
+
+# venv
+.venv
+*.so

bytepiece-py/.python-version

@@ -0,0 +1 @@
+3.12.2

bytepiece-py/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bytepiece-py"
-version = "0.2.0"
+version = "0.1.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -9,5 +9,5 @@ name = "bytepiece_py"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = "0.20.3"
-bytepiece = { workspace = true }
+pyo3 = "0.21"
+bytepiece = { workspace = true }

bytepiece-py/README.md

@@ -0,0 +1,3 @@
+# bytepiece-py
+
+Describe your project here.

bytepiece-py/pyproject.toml

@@ -1,16 +1,28 @@
-[build-system]
-requires = ["maturin>=1.1,<2.0"]
-build-backend = "maturin"
-
 [project]
 name = "bytepiece-py"
-requires-python = ">=3.7"
-classifiers = [
-    "Programming Language :: Rust",
-    "Programming Language :: Python :: Implementation :: CPython",
-    "Programming Language :: Python :: Implementation :: PyPy",
+version = "0.2.1"
+description = "Add your description here"
+authors = [
+    { name = "SunDoge", email = "[email protected]" }
 ]
+dependencies = []
+readme = "README.md"
+requires-python = ">= 3.8"
+
+[build-system]
+requires = ["maturin>=1.2,<2.0"]
+build-backend = "maturin"
 
+[tool.rye]
+managed = true
+dev-dependencies = [
+    "pip>=24.0",
+]
 
 [tool.maturin]
+python-source = "python"
+module-name = "bytepiece_py._lowlevel"
 features = ["pyo3/extension-module"]
+
+[tool.rye.scripts]
+dev = "maturin develop --skip-install"

bytepiece-py/bytepiece_py/tokenizer.py → bytepiece-py/python/bytepiece_py/__init__.py

@@ -1,9 +1,7 @@
 import unicodedata
 from typing import Dict, List, Tuple, Union
-from base64 import b64decode
-import json
 
-from . import bytepiece_py as _ext
+from bytepiece_py import _lowlevel
 
 
 def normalize(text: str) -> bytes:
@@ -13,9 +11,9 @@ def normalize(text: str) -> bytes:
 class Tokenizer:
     def __init__(self, pieces: Union[str, Dict[str, Tuple[str, int, str]]]) -> None:
         if isinstance(pieces, str):
-            self._tokenizer = _ext._Tokenizer.from_path(pieces)
+            self._tokenizer = _lowlevel._Tokenizer.from_path(pieces)
         else:
-            self._tokenizer = _ext._Tokenizer(pieces)
+            self._tokenizer = _lowlevel._Tokenizer(pieces)
 
     def encode(
         self,

bytepiece-py/requirements-dev.lock

@@ -0,0 +1,11 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+
+-e file:.
+pip==24.0

bytepiece-py/requirements.lock

@@ -0,0 +1,10 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+
+-e file:.

bytepiece-py/src/lib.rs

@@ -4,11 +4,9 @@ mod tokenizer;
 use pyo3::prelude::*;
 use tokenizer::_Tokenizer;
 
-pub use error::{Error, Result};
-
 /// A Python module implemented in Rust.
 #[pymodule]
-fn bytepiece_py(_py: Python, m: &PyModule) -> PyResult<()> {
+fn _lowlevel(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<_Tokenizer>()?;
     Ok(())
 }

bytepiece-py/src/tokenizer.rs

@@ -1,4 +1,4 @@
-use crate::Result;
+use crate::error::Result;
 use bytepiece::tokenizer::{make_owned_tokenizer, OwnedTokenizer, Pieces, Tokenize};
 use pyo3::prelude::*;
 use pyo3::types::{PyBytes, PyType};
@@ -19,7 +19,7 @@ impl _Tokenizer {
     }
 
     #[classmethod]
-    fn from_path(_cls: &PyType, path: &str) -> Result<Self> {
+    fn from_path(_cls: &Bound<PyType>, path: &str) -> Result<Self> {
         let tk = OwnedTokenizer::from_path(path)?;
         Ok(Self { inner: tk })
     }
@@ -29,17 +29,25 @@ impl _Tokenizer {
     }
 
     #[pyo3(signature = (text, alpha = -1.0))]
-    pub fn tokenize<'py>(&self, py: Python<'py>, text: &PyBytes, alpha: f64) -> Vec<&'py PyBytes> {
+    pub fn tokenize<'py>(
+        &self,
+        py: Python<'py>,
+        text: &Bound<PyBytes>,
+        alpha: f64,
+    ) -> Vec<Bound<'py, PyBytes>> {
         let bs = text.as_bytes();
         let tokens = py.allow_threads(|| self.inner.tokenize(&bs, alpha));
-        tokens.into_iter().map(|bs| PyBytes::new(py, bs)).collect()
+        tokens
+            .into_iter()
+            .map(|bs| PyBytes::new_bound(py, bs))
+            .collect()
     }
 
     #[pyo3(signature = (text, add_bos = false, add_eos = false, alpha = -1.0))]
     pub fn encode(
         &self,
         py: Python<'_>,
-        text: &PyBytes,
+        text: &Bound<PyBytes>,
         add_bos: bool,
         add_eos: bool,
         alpha: f64,
@@ -48,16 +56,16 @@ impl _Tokenizer {
         py.allow_threads(|| self.inner.encode(bs, add_bos, add_eos, alpha))
     }
 
-    pub fn decode<'py>(&self, py: Python<'py>, ids: Vec<usize>) -> Result<&'py PyBytes> {
+    pub fn decode<'py>(&self, py: Python<'py>, ids: Vec<usize>) -> Result<Bound<'py, PyBytes>> {
         let res = py.allow_threads(|| self.inner.decode(&ids))?;
-        Ok(PyBytes::new(py, &res))
+        Ok(PyBytes::new_bound(py, &res))
     }
 
-    pub fn id_to_piece<'py>(&self, py: Python<'py>, id: usize) -> &'py PyBytes {
-        PyBytes::new(py, self.inner.id_to_piece(id))
+    pub fn id_to_piece<'py>(&self, py: Python<'py>, id: usize) -> Bound<'py, PyBytes> {
+        PyBytes::new_bound(py, self.inner.id_to_piece(id))
     }
 
-    pub fn piece_to_id(&self, piece: &PyBytes) -> usize {
+    pub fn piece_to_id(&self, piece: &Bound<PyBytes>) -> usize {
         self.inner.piece_to_id(piece.as_bytes())
     }
 }