From 0903e39cc7046eec496b49a73841029f2d5cdbe2 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 3 Mar 2021 17:41:09 -0800 Subject: [PATCH 01/76] Remove obsolete api. --- src/py/api.rs | 3 +-- src/py/run_program.rs | 61 ------------------------------------------- src/py1/mod.rs | 8 ++++++ 3 files changed, 9 insertions(+), 63 deletions(-) create mode 100644 src/py1/mod.rs diff --git a/src/py/api.rs b/src/py/api.rs index 871132b6..e68c3d03 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -10,7 +10,7 @@ use super::glue::{_py_run_program, _serialize_from_bytes, _serialize_to_bytes}; use super::native_op_lookup::GenericNativeOpLookup; use super::py_node::PyNode; use super::run_program::{ - __pyo3_get_function_deserialize_and_run_program, __pyo3_get_function_serialize_and_run_program, + __pyo3_get_function_deserialize_and_run_program, __pyo3_get_function_serialized_length, STRICT_MODE, }; use crate::cost::Cost; @@ -124,7 +124,6 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(serialize_from_bytes, m)?)?; m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; - m.add_function(wrap_pyfunction!(serialize_and_run_program, m)?)?; m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; m.add("STRICT_MODE", STRICT_MODE)?; diff --git a/src/py/run_program.rs b/src/py/run_program.rs index 3e983af3..9caa0755 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -45,67 +45,6 @@ impl OperatorHandler for OperatorHandlerWithMode { } } -#[pyfunction] -pub fn serialize_and_run_program( - py: Python, - program: &[u8], - args: &[u8], - quote_kw: u8, - apply_kw: u8, - max_cost: Cost, - flags: u32, -) -> PyResult<(Cost, Py)> { - let mut opcode_lookup_by_name = HashMap::>::new(); - for (v, s) in [ - (4, "op_if"), - (5, "op_cons"), - (6, "op_first"), - (7, "op_rest"), - (8, "op_listp"), - (9, "op_raise"), - (10, "op_eq"), - (11, "op_sha256"), - (12, "op_add"), - (13, "op_subtract"), - (14, "op_multiply"), - (15, "op_divmod"), - (16, "op_substr"), - (17, "op_strlen"), - (18, "op_point_add"), - (19, "op_pubkey_for_exp"), - (20, "op_concat"), - (22, "op_gr"), - (23, "op_gr_bytes"), - (24, "op_logand"), - (25, "op_logior"), - (26, "op_logxor"), - (27, "op_lognot"), - (28, "op_ash"), - (29, "op_lsh"), - (30, "op_not"), - (31, "op_any"), - (32, "op_all"), - (33, "op_softfork"), - (34, "op_div"), - ] - .iter() - { - let v: Vec = vec![*v as u8]; - opcode_lookup_by_name.insert(s.to_string(), v); - } - - deserialize_and_run_program( - py, - program, - args, - quote_kw, - apply_kw, - opcode_lookup_by_name, - max_cost, - flags, - ) -} - #[allow(clippy::too_many_arguments)] #[pyfunction] pub fn deserialize_and_run_program( diff --git a/src/py1/mod.rs b/src/py1/mod.rs new file mode 100644 index 00000000..d4dca94b --- /dev/null +++ b/src/py1/mod.rs @@ -0,0 +1,8 @@ +pub mod api; +pub mod arc_allocator; +pub mod f_table; +pub mod glue; +pub mod native_op_lookup; +pub mod py_node; +pub mod run_program; +pub mod to_py_node; From b4a957ee276f6e0c3069188ff4b06ae438fb15bc Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 4 Mar 2021 00:27:37 -0800 Subject: [PATCH 02/76] First crack at gateway. Acked-by: Richard Kiss --- src/py/gateway.rs | 8 ++ src/py/int_allocator_gateway.rs | 194 ++++++++++++++++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 src/py/gateway.rs create mode 100644 src/py/int_allocator_gateway.rs diff --git a/src/py/gateway.rs b/src/py/gateway.rs new file mode 100644 index 00000000..6093f5ee --- /dev/null +++ b/src/py/gateway.rs @@ -0,0 +1,8 @@ +use pyo3::prelude::{PyObject, PyResult, Python}; + +use crate::allocator::Allocator; + +pub trait PythonGateway { + fn to_pyobject(self, py: Python, ptr: ::Ptr) -> PyResult; + fn from_pyobject(self, py: Python, o: PyObject) -> PyResult<::Ptr>; +} diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs new file mode 100644 index 00000000..9cd26a7d --- /dev/null +++ b/src/py/int_allocator_gateway.rs @@ -0,0 +1,194 @@ +use pyo3::{exceptions::PyBufferError, prelude::*}; +//use pyo3::prelude::{PyObject, PyResult, Python}; +use pyo3::types::PyTuple; +use pyo3::types::PyBytes; + +use crate::allocator::{Allocator, SExp}; +use crate::int_allocator::{IntAllocator, IntAtomBuf}; +use crate::reduction::EvalErr; + +use super::gateway::PythonGateway; + +#[pyclass(subclass, unsendable)] +pub struct PyCachingAllocator { + arena: PyObject, // &PyCell, + // TODO: cache created PyObjects +} + +/* +impl Allocator for &PyCachingAllocator { + type Ptr = i32; + type AtomBuf = IntAtomBuf; + + fn new_atom(&mut self, v: &[u8]) -> Result> { + self.arena.borrow_mut().arena.new_atom(v) + } + + fn new_pair( + &mut self, + first: Self::Ptr, + rest: Self::Ptr, + ) -> Result> { + self.arena.borrow_mut().arena.new_pair(first, rest) + } + + // create a new atom whose value is the given slice of the specified atom + fn new_substr( + &mut self, + node: Self::Ptr, + start: u32, + end: u32, + ) -> Result> { + self.arena.borrow_mut().arena.new_substr(node, start, end) + } + + fn atom<'a>(&'a self, node: &'a Self::Ptr) -> &'a [u8] { + self.arena.borrow().arena.atom(node) + } + + fn buf<'a>(&'a self, node: &'a Self::AtomBuf) -> &'a [u8] { + self.arena.borrow().arena.buf(node) + } + + fn sexp(&self, node: &Self::Ptr) -> SExp { + self.arena.borrow().arena.sexp(node) + } + + fn null(&self) -> Self::Ptr { + self.arena.borrow().arena.null() + } + + fn one(&self) -> Self::Ptr { + self.arena.borrow().arena.one() + } +} +*/ + +#[pyclass(subclass, unsendable)] +pub struct PyIntAllocator { + arena: IntAllocator, +} + +#[pyclass(subclass, unsendable)] +pub struct PyIntNode { + arena: PyObject, // &PyCell + ptr: ::Ptr, +} + +impl PyIntNode { + fn allocator<'p>(&'p self, py: Python<'p>) -> PyResult> { + let allocator: &PyCell = self.arena.extract(py)?; + Ok(allocator.try_borrow()?) + } + + fn allocator_mut<'p>(&'p self, py: Python<'p>) -> PyResult> { + let allocator: &PyCell = self.arena.extract(py)?; + Ok(allocator.try_borrow_mut()?) + } +} + +#[pymethods] +impl PyIntNode { + #[getter(pair)] + pub fn pair(&self, py: Python) -> PyResult> { + let allocator = self.allocator(py)?; + let allocator: &IntAllocator = &allocator.arena; + match allocator.sexp(&self.ptr) { + SExp::Pair(p1, p2) => { + { + let v: &PyTuple = PyTuple::new(py, &[p1, p2]); + let v: PyObject = v.into(); + Ok(Some(v)) + } + } + _ => Ok(None), + } + } + + /* + pub fn _pair(&self) -> Option<(PyNode, PyNode)> { + match ArcAllocator::new().sexp(&self.node) { + SExp::Pair(p1, p2) => Some((p1.into(), p2.into())), + _ => None, + } + } + */ + + #[getter(atom)] + pub fn atom(&self, py: Python) -> PyResult> { + let allocator = self.allocator(py)?; + let allocator: &IntAllocator = &allocator.arena; + match allocator.sexp(&self.ptr) { + SExp::Atom(atom) => { + let s: &[u8] = allocator.buf(&atom); + let s: &PyBytes = PyBytes::new(py, s); + let s: PyObject = s.into(); + Ok(Some(s)) + } + _ => Ok(None), + } + } +} + +/* +impl Allocator for &PyIntAllocator { + type Ptr = i32; + type AtomBuf = IntAtomBuf; + + fn new_atom(&mut self, v: &[u8]) -> Result> { + self.arena.new_atom(v) + } + + fn new_pair( + &mut self, + first: Self::Ptr, + rest: Self::Ptr, + ) -> Result> { + self.arena.new_pair(first, rest) + } + + // create a new atom whose value is the given slice of the specified atom + fn new_substr( + &mut self, + node: Self::Ptr, + start: u32, + end: u32, + ) -> Result> { + self.arena.new_substr(node, start, end) + } + + fn atom<'a>(&'a self, node: &'a Self::Ptr) -> &'a [u8] { + self.arena.atom(node) + } + + fn buf<'a>(&'a self, node: &'a Self::AtomBuf) -> &'a [u8] { + self.arena.buf(node) + } + + fn sexp(&self, node: &Self::Ptr) -> SExp { + self.arena.sexp(node) + } + + fn null(&self) -> Self::Ptr { + self.arena.null() + } + + fn one(&self) -> Self::Ptr { + self.arena.one() + } +} +*/ + +impl PythonGateway for &PyCachingAllocator { + fn to_pyobject(self, py: Python, ptr: ::Ptr) -> PyResult { + let arena = self.arena.clone(); + let node = PyIntNode { arena, ptr }; + let cell = PyCell::new(py, node)?; + Ok(cell.into_py(py)) + } + + fn from_pyobject(self, py: Python, o: PyObject) -> PyResult<::Ptr> { + let obj: &PyCell = o.extract(py)?; + Ok(obj.try_borrow()?.ptr) + } +} From 91fa184d73b356a2f4a7ef40ac00897b9dfcfafb Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 4 Mar 2021 13:55:27 -0800 Subject: [PATCH 03/76] Checkpoint. --- src/py/int_allocator_gateway.rs | 105 ++++++++++++++++++++++++++++---- 1 file changed, 94 insertions(+), 11 deletions(-) diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs index 9cd26a7d..779c149d 100644 --- a/src/py/int_allocator_gateway.rs +++ b/src/py/int_allocator_gateway.rs @@ -1,7 +1,9 @@ +use std::{borrow::Borrow, cell::Cell}; + use pyo3::{exceptions::PyBufferError, prelude::*}; //use pyo3::prelude::{PyObject, PyResult, Python}; -use pyo3::types::PyTuple; use pyo3::types::PyBytes; +use pyo3::types::PyTuple; use crate::allocator::{Allocator, SExp}; use crate::int_allocator::{IntAllocator, IntAtomBuf}; @@ -69,10 +71,36 @@ pub struct PyIntAllocator { arena: IntAllocator, } +struct PyView { + atom: PyObject, + pair: PyObject, +} + +impl PyView { + fn py_bytes<'p>(&'p self, py: Python<'p>) -> Option<&PyBytes> { + // this glue returns a &[u8] if self.atom has PyBytes behind it + let r: Option<&PyBytes> = self.atom.extract(py).ok(); + r + } + + fn py_pair<'p>( + &'p self, + py: Python<'p>, + ) -> Option<(&'p PyCell, &'p PyCell)> { + let args: &PyTuple = self.pair.extract(py).ok()?; + let p0: &'p PyCell = args.get_item(0).extract().unwrap(); + let p1: &'p PyCell = args.get_item(1).extract().unwrap(); + Some((p0, p1)) + } +} + #[pyclass(subclass, unsendable)] pub struct PyIntNode { arena: PyObject, // &PyCell - ptr: ::Ptr, + // rust view + native_view: Cell::Ptr>>, + // python view + py_view: Option, } impl PyIntNode { @@ -85,24 +113,69 @@ impl PyIntNode { let allocator: &PyCell = self.arena.extract(py)?; Ok(allocator.try_borrow_mut()?) } + + fn ptr(&mut self, py: Option) -> ::Ptr { + if let Some(r) = self.native_view.get() { + r.clone() + } else { + if let Some(py) = py { + self.ensure_native_view(py) + } else { + panic!("can't cast from python to native") + } + } + } + + fn ensure_native_view(&mut self, py: Python) -> ::Ptr { + let mut allocator = self.allocator_mut(py).unwrap(); + let mut allocator: &mut IntAllocator = &mut allocator.arena; + let mut to_cast: Vec<&PyIntNode> = vec![self]; + loop { + let t = to_cast.pop(); + match t { + None => break, + Some(t) => { + if t.native_view.get().is_none() { + let py_view = self.py_view.as_ref().unwrap(); + match py_view.py_bytes(py) { + Some(blob) => { + let new_ptr = allocator.new_atom(blob.as_bytes()).unwrap(); + t.native_view.set(Some(new_ptr)); + } + None => { + let (p1, p2) = py_view.py_pair(py).unwrap(); + } + } + } + } + } + } + self.native_view.get().unwrap() + } + + fn ensure_python_view(&self, py: Python) -> PyResult<&PyView> { + if self.py_view.is_none() {} + Ok(self.py_view.as_ref().unwrap()) + } } #[pymethods] impl PyIntNode { #[getter(pair)] - pub fn pair(&self, py: Python) -> PyResult> { + pub fn pair<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyObject> { + Ok(&self.ensure_python_view(py)?.pair) + + /* let allocator = self.allocator(py)?; let allocator: &IntAllocator = &allocator.arena; match allocator.sexp(&self.ptr) { SExp::Pair(p1, p2) => { - { - let v: &PyTuple = PyTuple::new(py, &[p1, p2]); - let v: PyObject = v.into(); - Ok(Some(v)) - } + let v: &PyTuple = PyTuple::new(py, &[p1, p2]); + let v: PyObject = v.into(); + Ok(Some(v)) } _ => Ok(None), - } + }*/ } /* @@ -115,7 +188,10 @@ impl PyIntNode { */ #[getter(atom)] - pub fn atom(&self, py: Python) -> PyResult> { + pub fn atom<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyObject> { + Ok(&self.ensure_python_view(py)?.atom) + + /* let allocator = self.allocator(py)?; let allocator: &IntAllocator = &allocator.arena; match allocator.sexp(&self.ptr) { @@ -127,6 +203,7 @@ impl PyIntNode { } _ => Ok(None), } + */ } } @@ -179,10 +256,15 @@ impl Allocator for &PyIntAllocator { } */ +/* impl PythonGateway for &PyCachingAllocator { fn to_pyobject(self, py: Python, ptr: ::Ptr) -> PyResult { let arena = self.arena.clone(); - let node = PyIntNode { arena, ptr }; + let node = PyIntNode { + arena, + native_view: Some(ptr), + py_view: None, + }; let cell = PyCell::new(py, node)?; Ok(cell.into_py(py)) } @@ -192,3 +274,4 @@ impl PythonGateway for &PyCachingAllocator { Ok(obj.try_borrow()?.ptr) } } +*/ From b7dd52f31793fa7161c35ce04197382f4d464374 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 4 Mar 2021 14:50:03 -0800 Subject: [PATCH 04/76] Checkpoint. --- src/py/int_allocator_gateway.rs | 63 +++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs index 779c149d..ac48102f 100644 --- a/src/py/int_allocator_gateway.rs +++ b/src/py/int_allocator_gateway.rs @@ -1,9 +1,12 @@ -use std::{borrow::Borrow, cell::Cell}; +use std::borrow::Borrow; +use std::cell::{Cell, Ref, RefCell}; use pyo3::{exceptions::PyBufferError, prelude::*}; //use pyo3::prelude::{PyObject, PyResult, Python}; +use pyo3::ffi::Py_None; use pyo3::types::PyBytes; use pyo3::types::PyTuple; +use pyo3::AsPyPointer; use crate::allocator::{Allocator, SExp}; use crate::int_allocator::{IntAllocator, IntAtomBuf}; @@ -77,7 +80,7 @@ struct PyView { } impl PyView { - fn py_bytes<'p>(&'p self, py: Python<'p>) -> Option<&PyBytes> { + fn py_bytes<'p>(&'p self, py: Python<'p>) -> Option<&'p PyBytes> { // this glue returns a &[u8] if self.atom has PyBytes behind it let r: Option<&PyBytes> = self.atom.extract(py).ok(); r @@ -100,7 +103,7 @@ pub struct PyIntNode { // rust view native_view: Cell::Ptr>>, // python view - py_view: Option, + py_view: RefCell>, } impl PyIntNode { @@ -136,7 +139,8 @@ impl PyIntNode { None => break, Some(t) => { if t.native_view.get().is_none() { - let py_view = self.py_view.as_ref().unwrap(); + let py_view = self.py_view.borrow(); + let py_view = py_view.as_ref().unwrap(); match py_view.py_bytes(py) { Some(blob) => { let new_ptr = allocator.new_atom(blob.as_bytes()).unwrap(); @@ -144,6 +148,7 @@ impl PyIntNode { } None => { let (p1, p2) = py_view.py_pair(py).unwrap(); + // TODO: finish this } } } @@ -153,17 +158,51 @@ impl PyIntNode { self.native_view.get().unwrap() } - fn ensure_python_view(&self, py: Python) -> PyResult<&PyView> { - if self.py_view.is_none() {} - Ok(self.py_view.as_ref().unwrap()) + fn ensure_python_view<'p>(&'p mut self, py: Python<'p>) -> PyResult>> { + if self.py_view.borrow().is_none() { + let mut allocator: &mut IntAllocator = &mut self.allocator_mut(py)?.arena; + let mut to_cast: Vec<&PyIntNode> = vec![self]; + loop { + let t = to_cast.pop(); + match t { + None => break, + Some(t) => { + if t.py_view.borrow().is_some() { + continue; + } + let ptr = t.native_view.get().unwrap(); + match allocator.sexp(&ptr) { + SExp::Atom(a) => { + let as_u8: &[u8] = allocator.buf(&a); + let py_bytes = PyBytes::new(py, as_u8); + let py_object: PyObject = py_bytes.to_object(py); + let py_view = PyView { + atom: py_object, + pair: ().to_object(py), + }; + t.py_view.replace(Some(py_view)); + } + SExp::Pair(p1, p2) => { + let r1 = allocator.sexp(&p1); + let r2 = allocator.sexp(&p2); + // TODO: finish this + } + } + } + } + } + } + Ok(self.py_view.borrow()) } } #[pymethods] impl PyIntNode { #[getter(pair)] - pub fn pair<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyObject> { - Ok(&self.ensure_python_view(py)?.pair) + pub fn pair<'p>(&'p mut self, py: Python<'p>) -> PyResult { + let t: Ref<'p, Option> = self.ensure_python_view(py)?; + let t1 = &t.as_ref().unwrap().pair; + Ok(t1.clone()) /* let allocator = self.allocator(py)?; @@ -188,8 +227,10 @@ impl PyIntNode { */ #[getter(atom)] - pub fn atom<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyObject> { - Ok(&self.ensure_python_view(py)?.atom) + pub fn atom<'p>(&'p mut self, py: Python<'p>) -> PyResult { + let t: Ref<'p, Option> = self.ensure_python_view(py)?; + let t1 = &t.as_ref().unwrap().atom; + Ok(t1.clone()) /* let allocator = self.allocator(py)?; From f0022416d16432a0b4cf110665b5323b1e162548 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 4 Mar 2021 23:22:06 -0800 Subject: [PATCH 05/76] Checkpoint. --- src/py/int_allocator_gateway.rs | 266 +++++++++++++------------------- 1 file changed, 104 insertions(+), 162 deletions(-) diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs index ac48102f..dca4966d 100644 --- a/src/py/int_allocator_gateway.rs +++ b/src/py/int_allocator_gateway.rs @@ -1,4 +1,3 @@ -use std::borrow::Borrow; use std::cell::{Cell, Ref, RefCell}; use pyo3::{exceptions::PyBufferError, prelude::*}; @@ -14,61 +13,6 @@ use crate::reduction::EvalErr; use super::gateway::PythonGateway; -#[pyclass(subclass, unsendable)] -pub struct PyCachingAllocator { - arena: PyObject, // &PyCell, - // TODO: cache created PyObjects -} - -/* -impl Allocator for &PyCachingAllocator { - type Ptr = i32; - type AtomBuf = IntAtomBuf; - - fn new_atom(&mut self, v: &[u8]) -> Result> { - self.arena.borrow_mut().arena.new_atom(v) - } - - fn new_pair( - &mut self, - first: Self::Ptr, - rest: Self::Ptr, - ) -> Result> { - self.arena.borrow_mut().arena.new_pair(first, rest) - } - - // create a new atom whose value is the given slice of the specified atom - fn new_substr( - &mut self, - node: Self::Ptr, - start: u32, - end: u32, - ) -> Result> { - self.arena.borrow_mut().arena.new_substr(node, start, end) - } - - fn atom<'a>(&'a self, node: &'a Self::Ptr) -> &'a [u8] { - self.arena.borrow().arena.atom(node) - } - - fn buf<'a>(&'a self, node: &'a Self::AtomBuf) -> &'a [u8] { - self.arena.borrow().arena.buf(node) - } - - fn sexp(&self, node: &Self::Ptr) -> SExp { - self.arena.borrow().arena.sexp(node) - } - - fn null(&self) -> Self::Ptr { - self.arena.borrow().arena.null() - } - - fn one(&self) -> Self::Ptr { - self.arena.borrow().arena.one() - } -} -*/ - #[pyclass(subclass, unsendable)] pub struct PyIntAllocator { arena: IntAllocator, @@ -117,51 +61,86 @@ impl PyIntNode { Ok(allocator.try_borrow_mut()?) } - fn ptr(&mut self, py: Option) -> ::Ptr { - if let Some(r) = self.native_view.get() { - r.clone() + fn ptr(slf: &PyCell, py: Option) -> ::Ptr { + if let Some(r) = slf.borrow().native_view.get() { + r } else { if let Some(py) = py { - self.ensure_native_view(py) + let p = slf.borrow(); + let mut allocator = p.allocator_mut(py).unwrap(); + let mut allocator: &mut IntAllocator = &mut allocator.arena; + + let mut to_cast: Vec = vec![slf.to_object(py)]; + + Self::ensure_native_view(to_cast, allocator, py); + slf.borrow().native_view.get().unwrap() } else { panic!("can't cast from python to native") } } } - fn ensure_native_view(&mut self, py: Python) -> ::Ptr { - let mut allocator = self.allocator_mut(py).unwrap(); - let mut allocator: &mut IntAllocator = &mut allocator.arena; - let mut to_cast: Vec<&PyIntNode> = vec![self]; - loop { - let t = to_cast.pop(); - match t { - None => break, - Some(t) => { - if t.native_view.get().is_none() { - let py_view = self.py_view.borrow(); - let py_view = py_view.as_ref().unwrap(); - match py_view.py_bytes(py) { - Some(blob) => { - let new_ptr = allocator.new_atom(blob.as_bytes()).unwrap(); - t.native_view.set(Some(new_ptr)); - } - None => { - let (p1, p2) = py_view.py_pair(py).unwrap(); - // TODO: finish this + fn ensure_native_view<'p>( + mut to_cast: Vec, + allocator: &mut IntAllocator, + py: Python<'p>, + ) -> () { + { + loop { + let t: Option = to_cast.pop(); + match t { + None => break, + Some(t0) => { + let t0_5 : &PyAny = t0.extract(py).unwrap(); + let t1: &PyCell = t0_5.downcast().unwrap(); + let t2: PyRef = t1.borrow(); + if t2.native_view.get().is_none() { + let py_view_ref: Ref> = t2.py_view.borrow(); + let py_view = py_view_ref.as_ref().unwrap(); + match py_view.py_bytes(py) { + Some(blob) => { + let new_ptr = allocator.new_atom(blob.as_bytes()).unwrap(); + t2.native_view.set(Some(new_ptr)); + } + None => { + let (p1, p2) = py_view.py_pair(py).unwrap(); + // check if both p1 and p2 have native views + // if so build and cache native view for t + let r1: Option<::Ptr> = + p1.borrow().native_view.get(); + let r2: Option<::Ptr> = + p2.borrow().native_view.get(); + if r1.is_some() && r2.is_some() { + let s1 = r1.unwrap(); + let s2 = r2.unwrap(); + let ptr = allocator.new_pair(s1, s2).unwrap(); + t2.native_view.set(Some(ptr)); + } else { + // otherwise, push t, push p1, push p2 back on stack to be processed + // + // UGH, these objects are type `PyCell` not &PyIntNode, what do I do + // + to_cast.push(p1.to_object(py)); + to_cast.push(p2.to_object(py)); + } + } } } } } } } - self.native_view.get().unwrap() } - fn ensure_python_view<'p>(&'p mut self, py: Python<'p>) -> PyResult>> { - if self.py_view.borrow().is_none() { - let mut allocator: &mut IntAllocator = &mut self.allocator_mut(py)?.arena; - let mut to_cast: Vec<&PyIntNode> = vec![self]; + fn ensure_python_view<'p>( + slf: &PyRef<'p, Self>, + py: Python<'p>, + //) -> PyResult>> { + ) -> PyResult<()> { + /* + if slf.py_view.borrow().is_none() { + let allocator: &'p IntAllocator = &slf.allocator(py).unwrap().arena; + let mut to_cast: Vec> = vec![slf]; loop { let t = to_cast.pop(); match t { @@ -183,24 +162,55 @@ impl PyIntNode { t.py_view.replace(Some(py_view)); } SExp::Pair(p1, p2) => { - let r1 = allocator.sexp(&p1); - let r2 = allocator.sexp(&p2); - // TODO: finish this + // create new n1, n2 child nodes of t + let arena = t.arena.clone(); + let native_view = Cell::new(Some(p1)); + let py_view = RefCell::new(None); + let n1 = PyCell::new( + py, + PyIntNode { + arena, + native_view, + py_view, + }, + )?; + let arena = t.arena.clone(); + let native_view = Cell::new(Some(p2)); + let py_view = RefCell::new(None); + let n2 = PyCell::new( + py, + PyIntNode { + arena, + native_view, + py_view, + }, + )?; + let py_object = PyTuple::new(py, [n1, n2]); + let py_view = PyView { + pair: py_object, + atom: ().to_object(py), + }; + t.py_view.replace(Some(py_view)); + to_cast.push(n1.borrow()); + to_cast.push(n2.borrow()); } } } } } } - Ok(self.py_view.borrow()) + //Ok(slf.py_view.borrow()) + */ + Ok(()) } } #[pymethods] impl PyIntNode { #[getter(pair)] - pub fn pair<'p>(&'p mut self, py: Python<'p>) -> PyResult { - let t: Ref<'p, Option> = self.ensure_python_view(py)?; + pub fn pair<'p>(slf: PyRef<'p, Self>, py: Python<'p>) -> PyResult { + Self::ensure_python_view(&slf, py)?; + let t: Ref> = slf.py_view.borrow(); let t1 = &t.as_ref().unwrap().pair; Ok(t1.clone()) @@ -227,8 +237,9 @@ impl PyIntNode { */ #[getter(atom)] - pub fn atom<'p>(&'p mut self, py: Python<'p>) -> PyResult { - let t: Ref<'p, Option> = self.ensure_python_view(py)?; + pub fn atom<'p>(slf: PyRef<'p, Self>, py: Python<'p>) -> PyResult { + Self::ensure_python_view(&slf, py)?; + let t: Ref> = slf.py_view.borrow(); let t1 = &t.as_ref().unwrap().atom; Ok(t1.clone()) @@ -247,72 +258,3 @@ impl PyIntNode { */ } } - -/* -impl Allocator for &PyIntAllocator { - type Ptr = i32; - type AtomBuf = IntAtomBuf; - - fn new_atom(&mut self, v: &[u8]) -> Result> { - self.arena.new_atom(v) - } - - fn new_pair( - &mut self, - first: Self::Ptr, - rest: Self::Ptr, - ) -> Result> { - self.arena.new_pair(first, rest) - } - - // create a new atom whose value is the given slice of the specified atom - fn new_substr( - &mut self, - node: Self::Ptr, - start: u32, - end: u32, - ) -> Result> { - self.arena.new_substr(node, start, end) - } - - fn atom<'a>(&'a self, node: &'a Self::Ptr) -> &'a [u8] { - self.arena.atom(node) - } - - fn buf<'a>(&'a self, node: &'a Self::AtomBuf) -> &'a [u8] { - self.arena.buf(node) - } - - fn sexp(&self, node: &Self::Ptr) -> SExp { - self.arena.sexp(node) - } - - fn null(&self) -> Self::Ptr { - self.arena.null() - } - - fn one(&self) -> Self::Ptr { - self.arena.one() - } -} -*/ - -/* -impl PythonGateway for &PyCachingAllocator { - fn to_pyobject(self, py: Python, ptr: ::Ptr) -> PyResult { - let arena = self.arena.clone(); - let node = PyIntNode { - arena, - native_view: Some(ptr), - py_view: None, - }; - let cell = PyCell::new(py, node)?; - Ok(cell.into_py(py)) - } - - fn from_pyobject(self, py: Python, o: PyObject) -> PyResult<::Ptr> { - let obj: &PyCell = o.extract(py)?; - Ok(obj.try_borrow()?.ptr) - } -} -*/ From 31a4b6a01664acd380c3607f33aceba0eb09d0a1 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 5 Mar 2021 13:03:37 -0800 Subject: [PATCH 06/76] Checkpoint. --- src/py/int_allocator_gateway.rs | 106 ++++++++++++++++---------------- 1 file changed, 54 insertions(+), 52 deletions(-) diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs index dca4966d..8e3cb724 100644 --- a/src/py/int_allocator_gateway.rs +++ b/src/py/int_allocator_gateway.rs @@ -85,44 +85,42 @@ impl PyIntNode { allocator: &mut IntAllocator, py: Python<'p>, ) -> () { - { - loop { - let t: Option = to_cast.pop(); - match t { - None => break, - Some(t0) => { - let t0_5 : &PyAny = t0.extract(py).unwrap(); - let t1: &PyCell = t0_5.downcast().unwrap(); - let t2: PyRef = t1.borrow(); - if t2.native_view.get().is_none() { - let py_view_ref: Ref> = t2.py_view.borrow(); - let py_view = py_view_ref.as_ref().unwrap(); - match py_view.py_bytes(py) { - Some(blob) => { - let new_ptr = allocator.new_atom(blob.as_bytes()).unwrap(); - t2.native_view.set(Some(new_ptr)); - } - None => { - let (p1, p2) = py_view.py_pair(py).unwrap(); - // check if both p1 and p2 have native views - // if so build and cache native view for t - let r1: Option<::Ptr> = - p1.borrow().native_view.get(); - let r2: Option<::Ptr> = - p2.borrow().native_view.get(); - if r1.is_some() && r2.is_some() { - let s1 = r1.unwrap(); - let s2 = r2.unwrap(); - let ptr = allocator.new_pair(s1, s2).unwrap(); - t2.native_view.set(Some(ptr)); - } else { - // otherwise, push t, push p1, push p2 back on stack to be processed - // - // UGH, these objects are type `PyCell` not &PyIntNode, what do I do - // - to_cast.push(p1.to_object(py)); - to_cast.push(p2.to_object(py)); - } + loop { + let t: Option = to_cast.pop(); + match t { + None => break, + Some(t0) => { + let t0_5: &PyAny = t0.extract(py).unwrap(); + let t1: &PyCell = t0_5.downcast().unwrap(); + let t2: PyRef = t1.borrow(); + if t2.native_view.get().is_none() { + let py_view_ref: Ref> = t2.py_view.borrow(); + let py_view = py_view_ref.as_ref().unwrap(); + match py_view.py_bytes(py) { + Some(blob) => { + let new_ptr = allocator.new_atom(blob.as_bytes()).unwrap(); + t2.native_view.set(Some(new_ptr)); + } + None => { + let (p1, p2) = py_view.py_pair(py).unwrap(); + // check if both p1 and p2 have native views + // if so build and cache native view for t + let r1: Option<::Ptr> = + p1.borrow().native_view.get(); + let r2: Option<::Ptr> = + p2.borrow().native_view.get(); + if r1.is_some() && r2.is_some() { + let s1 = r1.unwrap(); + let s2 = r2.unwrap(); + let ptr = allocator.new_pair(s1, s2).unwrap(); + t2.native_view.set(Some(ptr)); + } else { + // otherwise, push t, push p1, push p2 back on stack to be processed + // + // UGH, these objects are type `PyCell` not &PyIntNode, what do I do + // + to_cast.push(p1.to_object(py)); + to_cast.push(p2.to_object(py)); } } } @@ -133,9 +131,9 @@ impl PyIntNode { } fn ensure_python_view<'p>( - slf: &PyRef<'p, Self>, + mut to_cast: Vec, + allocator: &mut IntAllocator, py: Python<'p>, - //) -> PyResult>> { ) -> PyResult<()> { /* if slf.py_view.borrow().is_none() { @@ -208,11 +206,14 @@ impl PyIntNode { #[pymethods] impl PyIntNode { #[getter(pair)] - pub fn pair<'p>(slf: PyRef<'p, Self>, py: Python<'p>) -> PyResult { - Self::ensure_python_view(&slf, py)?; - let t: Ref> = slf.py_view.borrow(); - let t1 = &t.as_ref().unwrap().pair; - Ok(t1.clone()) + pub fn pair<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { + let t0: PyRef = slf.borrow(); + let mut t1: PyRefMut = t0.allocator_mut(py)?; + let allocator: &mut IntAllocator = &mut t1.arena; + Self::ensure_python_view(vec![slf.to_object(py)], allocator, py)?; + let t2: Ref> = t0.py_view.borrow(); + let t3 = &t2.as_ref().unwrap().pair; + Ok(t3.clone()) /* let allocator = self.allocator(py)?; @@ -235,14 +236,15 @@ impl PyIntNode { } } */ - #[getter(atom)] - pub fn atom<'p>(slf: PyRef<'p, Self>, py: Python<'p>) -> PyResult { - Self::ensure_python_view(&slf, py)?; - let t: Ref> = slf.py_view.borrow(); - let t1 = &t.as_ref().unwrap().atom; - Ok(t1.clone()) - + pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { + let t0: PyRef = slf.borrow(); + let mut t1: PyRefMut = t0.allocator_mut(py)?; + let allocator: &mut IntAllocator = &mut t1.arena; + Self::ensure_python_view(vec![slf.to_object(py)], allocator, py)?; + let t2: Ref> = t0.py_view.borrow(); + let t3 = &t2.as_ref().unwrap().atom; + Ok(t3.clone()) /* let allocator = self.allocator(py)?; let allocator: &IntAllocator = &allocator.arena; From a0a1c6978187e1b50c6d6c35114d475aa3966198 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 5 Mar 2021 13:10:07 -0800 Subject: [PATCH 07/76] Checkpoint. --- src/py/int_allocator_gateway.rs | 113 ++++++++++++++++---------------- 1 file changed, 55 insertions(+), 58 deletions(-) diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs index 8e3cb724..10565f9b 100644 --- a/src/py/int_allocator_gateway.rs +++ b/src/py/int_allocator_gateway.rs @@ -135,70 +135,67 @@ impl PyIntNode { allocator: &mut IntAllocator, py: Python<'p>, ) -> PyResult<()> { - /* - if slf.py_view.borrow().is_none() { - let allocator: &'p IntAllocator = &slf.allocator(py).unwrap().arena; - let mut to_cast: Vec> = vec![slf]; - loop { - let t = to_cast.pop(); - match t { - None => break, - Some(t) => { - if t.py_view.borrow().is_some() { - continue; + loop { + let t = to_cast.pop(); + match t { + None => break, + Some(t0) => { + let t1: &PyAny = t0.extract(py).unwrap(); + let t2: &PyCell = t1.downcast().unwrap(); + let t3: PyRef = t2.borrow(); + + if t3.py_view.borrow().is_some() { + continue; + } + let ptr = t3.native_view.get().unwrap(); + match allocator.sexp(&ptr) { + SExp::Atom(a) => { + let as_u8: &[u8] = allocator.buf(&a); + let py_bytes = PyBytes::new(py, as_u8); + let py_object: PyObject = py_bytes.to_object(py); + let py_view = PyView { + atom: py_object, + pair: ().to_object(py), + }; + t3.py_view.replace(Some(py_view)); } - let ptr = t.native_view.get().unwrap(); - match allocator.sexp(&ptr) { - SExp::Atom(a) => { - let as_u8: &[u8] = allocator.buf(&a); - let py_bytes = PyBytes::new(py, as_u8); - let py_object: PyObject = py_bytes.to_object(py); - let py_view = PyView { - atom: py_object, - pair: ().to_object(py), - }; - t.py_view.replace(Some(py_view)); - } - SExp::Pair(p1, p2) => { - // create new n1, n2 child nodes of t - let arena = t.arena.clone(); - let native_view = Cell::new(Some(p1)); - let py_view = RefCell::new(None); - let n1 = PyCell::new( - py, - PyIntNode { - arena, - native_view, - py_view, - }, - )?; - let arena = t.arena.clone(); - let native_view = Cell::new(Some(p2)); - let py_view = RefCell::new(None); - let n2 = PyCell::new( - py, - PyIntNode { - arena, - native_view, - py_view, - }, - )?; - let py_object = PyTuple::new(py, [n1, n2]); - let py_view = PyView { - pair: py_object, - atom: ().to_object(py), - }; - t.py_view.replace(Some(py_view)); - to_cast.push(n1.borrow()); - to_cast.push(n2.borrow()); - } + SExp::Pair(p1, p2) => { + // create new n1, n2 child nodes of t + let arena = t3.arena.clone(); + let native_view = Cell::new(Some(p1)); + let py_view = RefCell::new(None); + let n1 = PyCell::new( + py, + PyIntNode { + arena, + native_view, + py_view, + }, + )?; + let arena = t3.arena.clone(); + let native_view = Cell::new(Some(p2)); + let py_view = RefCell::new(None); + let n2 = PyCell::new( + py, + PyIntNode { + arena, + native_view, + py_view, + }, + )?; + let py_object = PyTuple::new(py, &[n1, n2]); + let py_view = PyView { + pair: py_object.to_object(py), + atom: ().to_object(py), + }; + t3.py_view.replace(Some(py_view)); + to_cast.push(n1.to_object(py)); + to_cast.push(n2.to_object(py)); } } } } } - //Ok(slf.py_view.borrow()) - */ Ok(()) } } From f23452c7bd08251a0941618f5240385bd178cecd Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 5 Mar 2021 14:04:57 -0800 Subject: [PATCH 08/76] First crack at returning `PyIntNode`. --- src/py/api.rs | 4 ++++ src/py/int_allocator_gateway.rs | 40 ++++++++++++++------------------- src/py/mod.rs | 1 + src/py/run_program.rs | 17 ++++++++++---- 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index e68c3d03..8d262727 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -7,6 +7,7 @@ use pyo3::PyObject; use super::arc_allocator::ArcAllocator; use super::glue::{_py_run_program, _serialize_from_bytes, _serialize_to_bytes}; +use super::int_allocator_gateway::{PyIntAllocator, PyIntNode}; use super::native_op_lookup::GenericNativeOpLookup; use super::py_node::PyNode; use super::run_program::{ @@ -130,6 +131,9 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(raise_eval_error, m)?)?; m.add_function(wrap_pyfunction!(serialized_length, m)?)?; diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs index 10565f9b..fc42d222 100644 --- a/src/py/int_allocator_gateway.rs +++ b/src/py/int_allocator_gateway.rs @@ -1,24 +1,18 @@ use std::cell::{Cell, Ref, RefCell}; -use pyo3::{exceptions::PyBufferError, prelude::*}; -//use pyo3::prelude::{PyObject, PyResult, Python}; -use pyo3::ffi::Py_None; +use pyo3::prelude::*; use pyo3::types::PyBytes; use pyo3::types::PyTuple; -use pyo3::AsPyPointer; use crate::allocator::{Allocator, SExp}; -use crate::int_allocator::{IntAllocator, IntAtomBuf}; -use crate::reduction::EvalErr; - -use super::gateway::PythonGateway; +use crate::int_allocator::IntAllocator; #[pyclass(subclass, unsendable)] pub struct PyIntAllocator { - arena: IntAllocator, + pub arena: IntAllocator, } -struct PyView { +pub struct PyView { atom: PyObject, pair: PyObject, } @@ -43,11 +37,11 @@ impl PyView { #[pyclass(subclass, unsendable)] pub struct PyIntNode { - arena: PyObject, // &PyCell + pub arena: PyObject, // &PyCell // rust view - native_view: Cell::Ptr>>, + pub native_view: Cell::Ptr>>, // python view - py_view: RefCell>, + pub py_view: RefCell>, } impl PyIntNode { @@ -80,11 +74,7 @@ impl PyIntNode { } } - fn ensure_native_view<'p>( - mut to_cast: Vec, - allocator: &mut IntAllocator, - py: Python<'p>, - ) -> () { + fn ensure_native_view(mut to_cast: Vec, allocator: &mut IntAllocator, py: Python) { loop { let t: Option = to_cast.pop(); match t { @@ -109,9 +99,7 @@ impl PyIntNode { p1.borrow().native_view.get(); let r2: Option<::Ptr> = p2.borrow().native_view.get(); - if r1.is_some() && r2.is_some() { - let s1 = r1.unwrap(); - let s2 = r2.unwrap(); + if let (Some(s1), Some(s2)) = (r1, r2) { let ptr = allocator.new_pair(s1, s2).unwrap(); t2.native_view.set(Some(ptr)); } else { @@ -130,10 +118,10 @@ impl PyIntNode { } } - fn ensure_python_view<'p>( + fn ensure_python_view( mut to_cast: Vec, allocator: &mut IntAllocator, - py: Python<'p>, + py: Python, ) -> PyResult<()> { loop { let t = to_cast.pop(); @@ -202,6 +190,11 @@ impl PyIntNode { #[pymethods] impl PyIntNode { + #[getter(arena)] + pub fn get_arena(&self) -> PyObject { + self.arena.clone() + } + #[getter(pair)] pub fn pair<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { let t0: PyRef = slf.borrow(); @@ -256,4 +249,5 @@ impl PyIntNode { } */ } + } diff --git a/src/py/mod.rs b/src/py/mod.rs index d4dca94b..0ed617dc 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -2,6 +2,7 @@ pub mod api; pub mod arc_allocator; pub mod f_table; pub mod glue; +pub mod int_allocator_gateway; pub mod native_op_lookup; pub mod py_node; pub mod run_program; diff --git a/src/py/run_program.rs b/src/py/run_program.rs index 9caa0755..faa30428 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -1,3 +1,4 @@ +use std::cell::{Cell, RefCell}; use std::collections::HashMap; use crate::allocator::Allocator; @@ -7,6 +8,7 @@ use crate::int_allocator::IntAllocator; use crate::more_ops::op_unknown; use crate::node::Node; use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; +use crate::py::int_allocator_gateway::{PyIntAllocator, PyIntNode}; use crate::reduction::Response; use crate::run_program::{run_program, OperatorHandler}; use crate::serialize::{node_from_bytes, node_to_bytes, serialized_length_from_bytes}; @@ -56,7 +58,7 @@ pub fn deserialize_and_run_program( opcode_lookup_by_name: HashMap>, max_cost: Cost, flags: u32, -) -> PyResult<(Cost, Py)> { +) -> PyResult<(Cost, PyObject)> { let mut allocator = IntAllocator::new(); let f_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); let strict: bool = (flags & STRICT_MODE) != 0; @@ -79,9 +81,16 @@ pub fn deserialize_and_run_program( }); match r { Ok(reduction) => { - let node_as_blob = node_to_bytes(&Node::new(&allocator, reduction.1))?; - let node_as_bytes: Py = PyBytes::new(py, &node_as_blob).into(); - Ok((reduction.0, node_as_bytes)) + let py_int_allocator = PyCell::new(py, PyIntAllocator { arena: allocator })?; + let py_int_node = PyCell::new( + py, + PyIntNode { + arena: py_int_allocator.to_object(py), + py_view: RefCell::new(None), + native_view: Cell::new(Some(reduction.1)), + }, + )?; + Ok((reduction.0, py_int_node.to_object(py))) } Err(eval_err) => { let node_as_blob = node_to_bytes(&Node::new(&allocator, eval_err.0))?; From f24ae7b1c316171b9ce9f61cbad70ef7eb71551b Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 5 Mar 2021 18:59:47 -0800 Subject: [PATCH 09/76] More methods. --- src/py/int_allocator_gateway.rs | 72 ++++++++++++++++++++++++++++++--- src/py/op_fn.rs | 0 2 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 src/py/op_fn.rs diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs index fc42d222..7ffa4f0f 100644 --- a/src/py/int_allocator_gateway.rs +++ b/src/py/int_allocator_gateway.rs @@ -3,6 +3,7 @@ use std::cell::{Cell, Ref, RefCell}; use pyo3::prelude::*; use pyo3::types::PyBytes; use pyo3::types::PyTuple; +use pyo3::types::PyType; use crate::allocator::{Allocator, SExp}; use crate::int_allocator::IntAllocator; @@ -18,6 +19,12 @@ pub struct PyView { } impl PyView { + pub fn new(atom: &PyObject, pair: &PyObject) -> Self { + let atom = atom.clone(); + let pair = pair.clone(); + PyView { atom, pair } + } + fn py_bytes<'p>(&'p self, py: Python<'p>) -> Option<&'p PyBytes> { // this glue returns a &[u8] if self.atom has PyBytes behind it let r: Option<&PyBytes> = self.atom.extract(py).ok(); @@ -45,6 +52,29 @@ pub struct PyIntNode { } impl PyIntNode { + pub fn new(arena: PyObject, native_view: Option, py_view: Option) -> Self { + let native_view = Cell::new(native_view); + let py_view = RefCell::new(py_view); + Self { + arena, + native_view, + py_view, + } + } + + pub fn from_ptr<'p>( + py: Python<'p>, + allocator: IntAllocator, + ptr: ::Ptr, + ) -> PyResult<&'p PyCell> { + let py_int_allocator = PyCell::new(py, PyIntAllocator { arena: allocator })?; + let py_int_node = PyCell::new( + py, + PyIntNode::new(py_int_allocator.to_object(py), Some(ptr), None), + ); + py_int_node + } + fn allocator<'p>(&'p self, py: Python<'p>) -> PyResult> { let allocator: &PyCell = self.arena.extract(py)?; Ok(allocator.try_borrow()?) @@ -55,7 +85,7 @@ impl PyIntNode { Ok(allocator.try_borrow_mut()?) } - fn ptr(slf: &PyCell, py: Option) -> ::Ptr { + pub fn ptr(slf: &PyCell, py: Option) -> ::Ptr { if let Some(r) = slf.borrow().native_view.get() { r } else { @@ -74,6 +104,21 @@ impl PyIntNode { } } + /* + pub fn get_py_view<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult>> { + let t0: PyRef = slf.borrow(); + { + let t1: Ref> = t0.py_view.borrow(); + if t1.is_none() { + let mut t2: PyRefMut = t0.allocator_mut(py)?; + let allocator: &mut IntAllocator = &mut t2.arena; + Self::ensure_python_view(vec![slf.to_object(py)], allocator, py)?; + } + } + Ok(t0.py_view.borrow()) + } + */ + fn ensure_native_view(mut to_cast: Vec, allocator: &mut IntAllocator, py: Python) { loop { let t: Option = to_cast.pop(); @@ -104,9 +149,6 @@ impl PyIntNode { t2.native_view.set(Some(ptr)); } else { // otherwise, push t, push p1, push p2 back on stack to be processed - // - // UGH, these objects are type `PyCell` not &PyIntNode, what do I do - // to_cast.push(p1.to_object(py)); to_cast.push(p2.to_object(py)); } @@ -190,6 +232,27 @@ impl PyIntNode { #[pymethods] impl PyIntNode { + #[classmethod] + fn new_atom(cls: &PyType, py: Python, atom: &PyBytes) -> PyResult { + let none: PyObject = py.eval("None", None, None)?.to_object(py); + let py_view = Some(PyView::new(&atom.to_object(py), &none)); + Ok(PyIntNode::new(none, None, py_view)) + } + + #[classmethod] + fn new_pair( + cls: &PyType, + py: Python, + p1: &PyCell, + p2: &PyCell, + ) -> PyResult { + let pair: &PyTuple = PyTuple::new(py, &[p1, p2]); + let none: PyObject = py.eval("None", None, None)?.to_object(py); + // TODO: ensure `pair` is a tuple of two `PyIntNode` + let py_view = Some(PyView::new(&none, &pair.to_object(py))); + Ok(PyIntNode::new(none, None, py_view)) + } + #[getter(arena)] pub fn get_arena(&self) -> PyObject { self.arena.clone() @@ -249,5 +312,4 @@ impl PyIntNode { } */ } - } diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs new file mode 100644 index 00000000..e69de29b From c7a9287fc71c3e9aeec847b6fbfcb8a739fe88b1 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 5 Mar 2021 19:22:08 -0800 Subject: [PATCH 10/76] Checkpoint. --- src/py/int_allocator_gateway.rs | 8 ++- src/py/mod.rs | 1 + src/py/op_fn.rs | 108 ++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 2 deletions(-) diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs index 7ffa4f0f..ff20f5aa 100644 --- a/src/py/int_allocator_gateway.rs +++ b/src/py/int_allocator_gateway.rs @@ -119,7 +119,11 @@ impl PyIntNode { } */ - fn ensure_native_view(mut to_cast: Vec, allocator: &mut IntAllocator, py: Python) { + pub fn ensure_native_view( + mut to_cast: Vec, + allocator: &mut IntAllocator, + py: Python, + ) { loop { let t: Option = to_cast.pop(); match t { @@ -160,7 +164,7 @@ impl PyIntNode { } } - fn ensure_python_view( + pub fn ensure_python_view( mut to_cast: Vec, allocator: &mut IntAllocator, py: Python, diff --git a/src/py/mod.rs b/src/py/mod.rs index 0ed617dc..ee12dbe6 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -4,6 +4,7 @@ pub mod f_table; pub mod glue; pub mod int_allocator_gateway; pub mod native_op_lookup; +pub mod op_fn; pub mod py_node; pub mod run_program; pub mod to_py_node; diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index e69de29b..9b43dcba 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -0,0 +1,108 @@ +use std::collections::HashMap; + +use pyo3::prelude::pyclass; +use pyo3::PyCell; +use pyo3::PyObject; +use pyo3::Python; +use pyo3::ToPyObject; + +use crate::allocator::Allocator; +use crate::cost::Cost; +use crate::int_allocator::IntAllocator; +use crate::py::f_table::FLookup; +use crate::py::int_allocator_gateway::PyIntNode; +use crate::reduction::{EvalErr, Reduction, Response}; +use crate::run_program::OperatorHandler; + +use super::py_node; + +#[pyclass] +struct PyOpFn { + callable: PyObject, +} + +struct PyOperatorHandler { + arena: PyObject, + native_lookup: FLookup, + //native_callable: HashMap, Box>, + py_callable: HashMap, PyObject>, +} + +impl OperatorHandler for PyOperatorHandler { + fn op( + &self, + allocator: &mut IntAllocator, + op_buf: ::AtomBuf, + args: &::Ptr, + max_cost: Cost, + ) -> Response<::Ptr> { + let op = allocator.buf(&op_buf); + if op.len() == 1 { + if let Some(f) = self.native_lookup[op[0] as usize] { + return f(allocator, args.clone(), max_cost); + } + } + + let py_obj = self.py_callable.get(op); + if let Some(obj) = py_obj { + return self.invoke_py_obj(obj.clone(), allocator, op_buf, args, max_cost); + }; + + Ok(Reduction(0, 0)) + } +} + +impl PyOperatorHandler { + fn invoke_py_obj( + &self, + obj: PyObject, + allocator: &mut IntAllocator, + op_buf: ::AtomBuf, + args: &::Ptr, + max_cost: Cost, + ) -> Response<::Ptr> { + + Python::with_gil(|py| { + let op = allocator.buf(&op_buf).to_object(py); + let py_int_node: &PyCell = + PyCell::new(py, PyIntNode::new(self.arena.clone(), Some(*args), None)).unwrap(); + + // this hack ensures we have python representations in all children + PyIntNode::ensure_python_view(vec![py_int_node.to_object(py)], allocator, py).unwrap(); + let r1 = obj.call1(py, (op, py_int_node)); + /* + match r1 { + Err(pyerr) => { + let eval_err: PyResult::Ptr>> = + eval_err_for_pyerr(py, &pyerr); + let r: EvalErr<::Ptr> = + unwrap_or_eval_err(eval_err, argument_list, "unexpected exception")?; + Err(r) + } + Ok(o) => { + let pair: &PyTuple = + unwrap_or_eval_err(o.extract(py), argument_list, "expected tuple")?; + + let i0: u32 = unwrap_or_eval_err( + pair.get_item(0).extract(), + argument_list, + "expected u32", + )?; + + let py_node: N = unwrap_or_eval_err( + pair.get_item(1).extract(), + argument_list, + "expected node", + )?; + + let node: ::Ptr = py_node.into(); + Ok(Reduction(i0 as Cost, node)) + } + } + */ + Ok(Reduction(0, 0)) + }) + } +} + +//fn(&mut T, ::Ptr, Cost) -> Response<::Ptr>; From 262aeaec35e4988e07e222483efe28826e2b4b67 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 5 Mar 2021 19:56:15 -0800 Subject: [PATCH 11/76] Checkpoint. --- src/py/op_fn.rs | 63 ++++++++++++++++++++++++++----------------- src/py/run_program.rs | 2 +- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 9b43dcba..abccbe6c 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -1,8 +1,12 @@ use std::collections::HashMap; use pyo3::prelude::pyclass; +use pyo3::types::PyString; +use pyo3::types::PyTuple; use pyo3::PyCell; +use pyo3::PyErr; use pyo3::PyObject; +use pyo3::PyResult; use pyo3::Python; use pyo3::ToPyObject; @@ -61,7 +65,6 @@ impl PyOperatorHandler { args: &::Ptr, max_cost: Cost, ) -> Response<::Ptr> { - Python::with_gil(|py| { let op = allocator.buf(&op_buf).to_object(py); let py_int_node: &PyCell = @@ -70,39 +73,51 @@ impl PyOperatorHandler { // this hack ensures we have python representations in all children PyIntNode::ensure_python_view(vec![py_int_node.to_object(py)], allocator, py).unwrap(); let r1 = obj.call1(py, (op, py_int_node)); - /* + match r1 { Err(pyerr) => { - let eval_err: PyResult::Ptr>> = - eval_err_for_pyerr(py, &pyerr); - let r: EvalErr<::Ptr> = - unwrap_or_eval_err(eval_err, argument_list, "unexpected exception")?; + let eval_err: PyResult> = eval_err_for_pyerr(py, &pyerr); + let r: EvalErr = + unwrap_or_eval_err(eval_err, args, "unexpected exception")?; Err(r) } Ok(o) => { - let pair: &PyTuple = - unwrap_or_eval_err(o.extract(py), argument_list, "expected tuple")?; - - let i0: u32 = unwrap_or_eval_err( - pair.get_item(0).extract(), - argument_list, - "expected u32", - )?; - - let py_node: N = unwrap_or_eval_err( - pair.get_item(1).extract(), - argument_list, - "expected node", - )?; - - let node: ::Ptr = py_node.into(); + let pair: &PyTuple = unwrap_or_eval_err(o.extract(py), args, "expected tuple")?; + + let i0: u32 = + unwrap_or_eval_err(pair.get_item(0).extract(), args, "expected u32")?; + + let py_node: &PyCell = + unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; + + let node: i32 = PyIntNode::ptr(py_node, Some(py)); Ok(Reduction(i0 as Cost, node)) } } - */ - Ok(Reduction(0, 0)) }) } } //fn(&mut T, ::Ptr, Cost) -> Response<::Ptr>; + +/// turn a `PyErr` into an `EvalErr

` if at all possible +/// otherwise, return a `PyErr` + +fn eval_err_for_pyerr<'p>(py: Python<'p>, pyerr: &PyErr) -> PyResult> { + let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; + let arg0: &PyString = args.get_item(0).extract()?; + let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; + let node: i32 = PyIntNode::ptr(&sexp, Some(py)); + let s: String = arg0.to_str()?.to_string(); + Ok(EvalErr(node, s)) +} + +fn unwrap_or_eval_err(obj: PyResult, err_node: &P, msg: &str) -> Result> +where + P: Clone, +{ + match obj { + Err(_py_err) => Err(EvalErr(err_node.clone(), msg.to_string())), + Ok(o) => Ok(o), + } +} diff --git a/src/py/run_program.rs b/src/py/run_program.rs index faa30428..a2562b63 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -18,7 +18,7 @@ use pyo3::types::{PyBytes, PyDict}; pub const STRICT_MODE: u32 = 1; -struct OperatorHandlerWithMode { +pub struct OperatorHandlerWithMode { f_lookup: FLookup, strict: bool, } From 858328210d54b228002c82513834ece6864c3247 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 9 Mar 2021 13:38:54 -0800 Subject: [PATCH 12/76] Checkpoint. Yikes. --- src/py/api.rs | 127 ++++++++++++---- src/py/int_allocator_gateway.rs | 48 +++--- src/py/mod.rs | 4 + src/py/native_op_lookup.rs | 3 + src/py/native_view.rs | 16 ++ src/py/op_fn.rs | 115 +++++++++----- src/py/py_int_allocator.rs | 1 + src/py/py_na_node.rs | 259 ++++++++++++++++++++++++++++++++ src/py/py_view.rs | 49 ++++++ 9 files changed, 527 insertions(+), 95 deletions(-) create mode 100644 src/py/native_view.rs create mode 100644 src/py/py_int_allocator.rs create mode 100644 src/py/py_na_node.rs create mode 100644 src/py/py_view.rs diff --git a/src/py/api.rs b/src/py/api.rs index 8d262727..1a45a1ee 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -5,20 +5,23 @@ use pyo3::types::{PyDict, PyString}; use pyo3::wrap_pyfunction; use pyo3::PyObject; -use super::arc_allocator::ArcAllocator; -use super::glue::{_py_run_program, _serialize_from_bytes, _serialize_to_bytes}; +use super::glue::{_serialize_from_bytes, _serialize_to_bytes}; use super::int_allocator_gateway::{PyIntAllocator, PyIntNode}; use super::native_op_lookup::GenericNativeOpLookup; -use super::py_node::PyNode; use super::run_program::{ - __pyo3_get_function_deserialize_and_run_program, - __pyo3_get_function_serialized_length, STRICT_MODE, + __pyo3_get_function_deserialize_and_run_program, __pyo3_get_function_serialized_length, + STRICT_MODE, }; -use crate::cost::Cost; +use crate::int_allocator::IntAllocator; +use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; +use crate::py::run_program::OperatorHandlerWithMode; +use crate::run_program::OperatorHandler; +use crate::{allocator, cost::Cost}; -type AllocatorT<'a> = ArcAllocator; -type NodeClass = PyNode; +type AllocatorT<'a> = IntAllocator; +type NodeClass = PyIntNode; +/* #[pyclass] pub struct NativeOpLookup { nol: usize, // Box>, @@ -43,46 +46,51 @@ impl Drop for NativeOpLookup { } impl NativeOpLookup { - fn new_from_gnol(gnol: Box>) -> Self { - Self { + fn new_from_gnol(gnol: Box>) -> Self { + NativeOpLookup { nol: Box::into_raw(gnol) as usize, } } } impl NativeOpLookup { - fn gnol<'a, 'p>(&self, _py: Python<'p>) -> &'a GenericNativeOpLookup, PyNode> { + fn gnol<'a, 'p>( + &self, + _py: Python<'p>, + ) -> &'a GenericNativeOpLookup, PyIntNode> { unsafe { &*(self.nol as *const GenericNativeOpLookup) } } } +*/ #[pyfunction] #[allow(clippy::too_many_arguments)] fn py_run_program( py: Python, - program: &NodeClass, - args: &NodeClass, + program: &PyCell, + args: &PyCell, quote_kw: u8, apply_kw: u8, max_cost: Cost, - op_lookup: Py, - pre_eval: PyObject, -) -> PyResult<(Cost, NodeClass)> { - let mut allocator = allocator_for_py(py); - let op_lookup: &PyCell = op_lookup.as_ref(py); - let op_lookup: PyRef = op_lookup.borrow(); - let op_lookup: Box> = - Box::new(op_lookup.gnol(py).to_owned()); + opcode_lookup_by_name: HashMap>, + py_callback: PyObject, + //op_lookup: Py, + //pre_eval: PyObject, +) -> PyResult<(Cost, PyObject)> { + //let f_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); + //let strict: bool = false; + //let f: Box> = + // Box::new(OperatorHandlerWithMode { f_lookup, strict }); + _py_run_program( py, - &mut allocator, program, args, quote_kw, apply_kw, max_cost, - op_lookup, - pre_eval, + opcode_lookup_by_name, + py_callback, ) } @@ -106,36 +114,91 @@ const fn allocator_for_py(_py: Python) -> AllocatorT { AllocatorT::new() } +/* #[pyfunction] -fn serialize_from_bytes(py: Python, blob: &[u8]) -> NodeClass { +fn serialize_from_bytes(py: Python, blob: &[u8]) -> PyIntNode { let mut allocator = allocator_for_py(py); - _serialize_from_bytes(&mut allocator, blob) + + let v = _serialize_from_bytes(&mut allocator, blob); + PyIntNode {} } #[pyfunction] fn serialize_to_bytes(py: Python, sexp: &PyAny) -> PyResult { let allocator = allocator_for_py(py); _serialize_to_bytes::(&allocator, py, sexp) -} +}*/ /// This module is a python module implemented in Rust. #[pymodule] fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(py_run_program, m)?)?; - m.add_function(wrap_pyfunction!(serialize_from_bytes, m)?)?; - m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; + //m.add_function(wrap_pyfunction!(serialize_from_bytes, m)?)?; + //m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; m.add("STRICT_MODE", STRICT_MODE)?; - m.add_class::()?; - m.add_class::()?; + //m.add_class::()?; + // m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_function(wrap_pyfunction!(raise_eval_error, m)?)?; m.add_function(wrap_pyfunction!(serialized_length, m)?)?; Ok(()) } + +use crate::py::op_fn::PyOperatorHandler; +use crate::reduction::{EvalErr, Reduction}; + +#[allow(clippy::too_many_arguments)] +pub fn _py_run_program( + py: Python, + program: &PyCell, + args: &PyCell, + quote_kw: u8, + apply_kw: u8, + max_cost: Cost, + opcode_lookup_by_name: HashMap>, + py_callback: PyObject, +) -> PyResult<(Cost, PyObject)> { + let mut allocator = IntAllocator::new(); + let arena = PyCell::new(py, PyIntAllocator { arena: allocator })?; + let mut b = arena.borrow_mut(); + let mut allocator: &mut IntAllocator = &mut b.arena; + + println!("1"); + let program = PyIntNode::ptr(program, Some(py), arena.to_object(py), &mut allocator); + println!("2"); + let args = PyIntNode::ptr(args, Some(py), arena.to_object(py), &mut allocator); + println!("3"); + + let op_lookup = Box::new(PyOperatorHandler::new( + opcode_lookup_by_name, + arena.to_object(py), + py_callback, + )); + + let r: Result, EvalErr> = crate::run_program::run_program( + allocator, &program, &args, quote_kw, apply_kw, max_cost, op_lookup, None, + ); + + match r { + Ok(reduction) => { + let r = PyIntNode::from_ptr(py, arena.to_object(py), reduction.1)?; + Ok((reduction.0, r.to_object(py))) + } + Err(eval_err) => { + let node: PyObject = eval_err.0.to_object(py); + let s: String = eval_err.1; + let s1: &str = &s; + let msg: &PyString = PyString::new(py, s1); + match raise_eval_error(py, &msg, node) { + Err(x) => Err(x), + _ => panic!(), + } + } + } +} diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs index ff20f5aa..006f6ec2 100644 --- a/src/py/int_allocator_gateway.rs +++ b/src/py/int_allocator_gateway.rs @@ -64,14 +64,10 @@ impl PyIntNode { pub fn from_ptr<'p>( py: Python<'p>, - allocator: IntAllocator, + py_int_allocator: PyObject, ptr: ::Ptr, ) -> PyResult<&'p PyCell> { - let py_int_allocator = PyCell::new(py, PyIntAllocator { arena: allocator })?; - let py_int_node = PyCell::new( - py, - PyIntNode::new(py_int_allocator.to_object(py), Some(ptr), None), - ); + let py_int_node = PyCell::new(py, PyIntNode::new(py_int_allocator, Some(ptr), None)); py_int_node } @@ -85,18 +81,20 @@ impl PyIntNode { Ok(allocator.try_borrow_mut()?) } - pub fn ptr(slf: &PyCell, py: Option) -> ::Ptr { + pub fn ptr( + slf: &PyCell, + py: Option, + arena: PyObject, + allocator: &mut IntAllocator, + ) -> ::Ptr { if let Some(r) = slf.borrow().native_view.get() { r } else { if let Some(py) = py { let p = slf.borrow(); - let mut allocator = p.allocator_mut(py).unwrap(); - let mut allocator: &mut IntAllocator = &mut allocator.arena; - let mut to_cast: Vec = vec![slf.to_object(py)]; - Self::ensure_native_view(to_cast, allocator, py); + Self::ensure_native_view(to_cast, arena, allocator, py); slf.borrow().native_view.get().unwrap() } else { panic!("can't cast from python to native") @@ -121,6 +119,7 @@ impl PyIntNode { pub fn ensure_native_view( mut to_cast: Vec, + arena: PyObject, allocator: &mut IntAllocator, py: Python, ) { @@ -131,7 +130,7 @@ impl PyIntNode { Some(t0) => { let t0_5: &PyAny = t0.extract(py).unwrap(); let t1: &PyCell = t0_5.downcast().unwrap(); - let t2: PyRef = t1.borrow(); + let mut t2: PyRefMut = t1.borrow_mut(); if t2.native_view.get().is_none() { let py_view_ref: Ref> = t2.py_view.borrow(); let py_view = py_view_ref.as_ref().unwrap(); @@ -151,6 +150,7 @@ impl PyIntNode { if let (Some(s1), Some(s2)) = (r1, r2) { let ptr = allocator.new_pair(s1, s2).unwrap(); t2.native_view.set(Some(ptr)); + t2.arena = arena.clone(); } else { // otherwise, push t, push p1, push p2 back on stack to be processed to_cast.push(p1.to_object(py)); @@ -265,11 +265,13 @@ impl PyIntNode { #[getter(pair)] pub fn pair<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { let t0: PyRef = slf.borrow(); - let mut t1: PyRefMut = t0.allocator_mut(py)?; - let allocator: &mut IntAllocator = &mut t1.arena; - Self::ensure_python_view(vec![slf.to_object(py)], allocator, py)?; - let t2: Ref> = t0.py_view.borrow(); - let t3 = &t2.as_ref().unwrap().pair; + let t1: Ref> = t0.py_view.borrow(); + if t1.is_none() { + let mut t2: PyRefMut = t0.allocator_mut(py)?; + let allocator: &mut IntAllocator = &mut t2.arena; + Self::ensure_python_view(vec![slf.to_object(py)], allocator, py)?; + } + let t3 = &t1.as_ref().unwrap().pair; Ok(t3.clone()) /* @@ -296,11 +298,13 @@ impl PyIntNode { #[getter(atom)] pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { let t0: PyRef = slf.borrow(); - let mut t1: PyRefMut = t0.allocator_mut(py)?; - let allocator: &mut IntAllocator = &mut t1.arena; - Self::ensure_python_view(vec![slf.to_object(py)], allocator, py)?; - let t2: Ref> = t0.py_view.borrow(); - let t3 = &t2.as_ref().unwrap().atom; + let t1: Ref> = t0.py_view.borrow(); + if t1.is_none() { + let mut t2: PyRefMut = t0.allocator_mut(py)?; + let allocator: &mut IntAllocator = &mut t2.arena; + Self::ensure_python_view(vec![slf.to_object(py)], allocator, py)?; + } + let t3 = &t1.as_ref().unwrap().atom; Ok(t3.clone()) /* let allocator = self.allocator(py)?; diff --git a/src/py/mod.rs b/src/py/mod.rs index ee12dbe6..656fa0a0 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -4,7 +4,11 @@ pub mod f_table; pub mod glue; pub mod int_allocator_gateway; pub mod native_op_lookup; +pub mod native_view; pub mod op_fn; +pub mod py_int_allocator; +pub mod py_na_node; pub mod py_node; +pub mod py_view; pub mod run_program; pub mod to_py_node; diff --git a/src/py/native_op_lookup.rs b/src/py/native_op_lookup.rs index e65366e8..73d537a5 100644 --- a/src/py/native_op_lookup.rs +++ b/src/py/native_op_lookup.rs @@ -14,6 +14,9 @@ use super::f_table::{f_lookup_for_hashmap, FLookup}; use super::to_py_node::ToPyNode; +/// turn a `PyErr` into an `EvalErr

` if at all possible +/// otherwise, return a `PyErr` + fn eval_err_for_pyerr<'s, 'p: 's, 'e: 's, P, N>( py: Python<'p>, pyerr: &'e PyErr, diff --git a/src/py/native_view.rs b/src/py/native_view.rs new file mode 100644 index 00000000..8f73125a --- /dev/null +++ b/src/py/native_view.rs @@ -0,0 +1,16 @@ +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyTuple}; + +use crate::allocator::Allocator; +use crate::int_allocator::IntAllocator; + +pub struct NativeView { + pub arena: PyObject, // PyCell, + pub ptr: ::Ptr, +} + +impl NativeView { + pub fn new(arena: PyObject, ptr: ::Ptr) -> Self { + NativeView { arena, ptr } + } +} diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index abccbe6c..1a082817 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -1,3 +1,4 @@ +use std::cell::RefCell; use std::collections::HashMap; use pyo3::prelude::pyclass; @@ -13,46 +14,35 @@ use pyo3::ToPyObject; use crate::allocator::Allocator; use crate::cost::Cost; use crate::int_allocator::IntAllocator; -use crate::py::f_table::FLookup; +use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; use crate::py::int_allocator_gateway::PyIntNode; use crate::reduction::{EvalErr, Reduction, Response}; use crate::run_program::OperatorHandler; use super::py_node; -#[pyclass] -struct PyOpFn { - callable: PyObject, -} - -struct PyOperatorHandler { - arena: PyObject, +pub struct PyOperatorHandler { native_lookup: FLookup, - //native_callable: HashMap, Box>, - py_callable: HashMap, PyObject>, + arena: PyObject, + py_callable: PyObject, + cache: RefCell>, } -impl OperatorHandler for PyOperatorHandler { - fn op( - &self, - allocator: &mut IntAllocator, - op_buf: ::AtomBuf, - args: &::Ptr, - max_cost: Cost, - ) -> Response<::Ptr> { - let op = allocator.buf(&op_buf); - if op.len() == 1 { - if let Some(f) = self.native_lookup[op[0] as usize] { - return f(allocator, args.clone(), max_cost); - } +impl PyOperatorHandler { + pub fn new( + opcode_lookup_by_name: HashMap>, + arena: PyObject, + py_callable: PyObject, + ) -> Self { + let native_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); + + let cache = RefCell::new(HashMap::new()); + PyOperatorHandler { + native_lookup, + arena, + py_callable, + cache, } - - let py_obj = self.py_callable.get(op); - if let Some(obj) = py_obj { - return self.invoke_py_obj(obj.clone(), allocator, op_buf, args, max_cost); - }; - - Ok(Reduction(0, 0)) } } @@ -60,6 +50,7 @@ impl PyOperatorHandler { fn invoke_py_obj( &self, obj: PyObject, + arena: PyObject, allocator: &mut IntAllocator, op_buf: ::AtomBuf, args: &::Ptr, @@ -67,16 +58,13 @@ impl PyOperatorHandler { ) -> Response<::Ptr> { Python::with_gil(|py| { let op = allocator.buf(&op_buf).to_object(py); - let py_int_node: &PyCell = - PyCell::new(py, PyIntNode::new(self.arena.clone(), Some(*args), None)).unwrap(); - - // this hack ensures we have python representations in all children - PyIntNode::ensure_python_view(vec![py_int_node.to_object(py)], allocator, py).unwrap(); + let py_int_node = self.uncache(py, allocator, args); let r1 = obj.call1(py, (op, py_int_node)); match r1 { Err(pyerr) => { - let eval_err: PyResult> = eval_err_for_pyerr(py, &pyerr); + let eval_err: PyResult> = + eval_err_for_pyerr(py, &pyerr, arena.clone(), allocator); let r: EvalErr = unwrap_or_eval_err(eval_err, args, "unexpected exception")?; Err(r) @@ -90,24 +78,69 @@ impl PyOperatorHandler { let py_node: &PyCell = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let node: i32 = PyIntNode::ptr(py_node, Some(py)); + let node: i32 = PyIntNode::ptr(py_node, Some(py), arena.clone(), allocator); Ok(Reduction(i0 as Cost, node)) } } }) } + + fn uncache(&self, py: Python, allocator: &mut IntAllocator, args: &i32) -> PyObject { + let args = args.clone(); + let mut cache = self.cache.borrow_mut(); + match cache.get(&args) { + Some(obj) => obj.clone(), + None => { + let py_int_node: &PyCell = + PyCell::new(py, PyIntNode::new(self.arena.clone(), Some(args), None)).unwrap(); + // this hack ensures we have python representations in all children + PyIntNode::ensure_python_view(vec![py_int_node.to_object(py)], allocator, py); + let obj: PyObject = py_int_node.to_object(py); + cache.insert(args, obj.clone()); + obj + } + } + } } -//fn(&mut T, ::Ptr, Cost) -> Response<::Ptr>; +impl OperatorHandler for PyOperatorHandler { + fn op( + &self, + allocator: &mut IntAllocator, + op_buf: ::AtomBuf, + args: &::Ptr, + max_cost: Cost, + ) -> Response<::Ptr> { + let op = allocator.buf(&op_buf); + if op.len() == 1 { + if let Some(f) = self.native_lookup[op[0] as usize] { + return f(allocator, args.clone(), max_cost); + } + } + + self.invoke_py_obj( + self.py_callable.clone(), + self.arena.clone(), + allocator, + op_buf, + args, + max_cost, + ) + } +} /// turn a `PyErr` into an `EvalErr

` if at all possible /// otherwise, return a `PyErr` - -fn eval_err_for_pyerr<'p>(py: Python<'p>, pyerr: &PyErr) -> PyResult> { +fn eval_err_for_pyerr<'p>( + py: Python<'p>, + pyerr: &PyErr, + arena: PyObject, + allocator: &mut IntAllocator, +) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = PyIntNode::ptr(&sexp, Some(py)); + let node: i32 = PyIntNode::ptr(&sexp, Some(py), arena, allocator); let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) } diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/py/py_int_allocator.rs @@ -0,0 +1 @@ + diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs new file mode 100644 index 00000000..6091ece5 --- /dev/null +++ b/src/py/py_na_node.rs @@ -0,0 +1,259 @@ +use std::borrow::Borrow; +use std::cell::{Ref, RefCell}; + +use pyo3::prelude::*; +use pyo3::types::{IntoPyDict, PyBytes, PyTuple, PyType}; + +use crate::allocator::{Allocator, SExp}; +use crate::int_allocator::IntAllocator; + +use super::native_view::NativeView; +use super::py_int_allocator::PyIntAllocator; +use super::py_view::PyView; + +enum View { + Python(PyView), + Native(NativeView), +} + +#[pyclass] +pub struct PyNaNode { + node: View, + int_arena_cache: PyObject, // WeakKeyDict[PyIntAllocator, int] +} + +impl PyNaNode { + fn new(py: Python, view: View) -> PyResult<&PyCell> { + let int_arena_cache = py + .eval("import weakref; weakref.WeakKeyDictionary()", None, None)? + .to_object(py); + + let node = PyNaNode { + node: view, + int_arena_cache, + }; + if let View::Native(native_view) = view { + node.add_to_arena_cache(py, &native_view.arena, native_view.ptr)?; + } + + let r = PyCell::new(py, node)?; + Ok(r) + } + + fn add_to_arena_cache( + &self, + py: Python, + arena: &PyObject, + ptr: ::Ptr, + ) -> PyResult<()> { + let locals = [ + ("cache", self.int_arena_cache.clone()), + ("key", arena.clone()), + ("value", ptr.to_object(py)), + ] + .into_py_dict(py); + + py.run("cache[key] = value", None, Some(locals))?; + Ok(()) + } + + pub fn from_ptr<'p>( + py: Python<'p>, + py_int_allocator: &PyObject, + ptr: ::Ptr, + ) -> PyResult<&'p PyCell> { + let native_view = NativeView::new(py_int_allocator.clone(), ptr); + Self::new(py, View::Native(native_view)) + } + + fn check_cache( + &self, + py: Python, + key: &PyObject, + ) -> PyResult::Ptr>> { + let locals = [ + ("cache", self.int_arena_cache.clone()), + ("key", key.clone()), + ] + .into_py_dict(py); + py.eval("cache.get(key)", None, Some(locals))?.extract() + } + + pub fn ptr( + slf: &PyCell, + py: Python, + arena: &PyObject, + allocator: &mut IntAllocator, + ) -> PyResult<::Ptr> { + { + let self_ref = slf.try_borrow()?; + match self_ref.check_cache(py, arena)? { + Some(v) => return Ok(v), + _ => (), + } + } + Self::ensure_native_view_cached(slf, py, arena, allocator)?; + Ok(slf.try_borrow()?.check_cache(py, arena)?.unwrap()) + } + + pub fn ensure_native_view_cached( + slf: &PyCell, + py: Python, + arena: &PyObject, + allocator: &mut IntAllocator, + ) -> PyResult<()> { + let mut to_cast: Vec<&PyCell> = vec![slf]; + Ok(loop { + let t: Option<&PyCell> = to_cast.pop(); + match t { + None => break, + Some(py_cell) => { + let transfer: Option<(&PyCell, &PyCell)> = py_cell + .try_borrow_mut()? + .add_to_native_cache(py, arena, allocator)?; + if let Some((p0, p1)) = transfer { + to_cast.push(p0); + to_cast.push(p1); + } + } + } + }) + } + + fn add_to_native_cache<'p>( + &'p mut self, + py: Python<'p>, + arena: &PyObject, + allocator: &mut IntAllocator, + ) -> PyResult, &'p PyCell)>> { + // if it's an atom, we add it to the allocator & cache the addition + // if it's a pair, and BOTH are in the cache, we add to allocator & cache + // otherwise, we return both children so they can be cached (if necessary) + if self.check_cache(py, arena)?.is_none() { + let py_view = self.ensure_python_view(py, Some((arena, allocator)))?; + let new_ptr = { + match py_view { + PyView::Atom(obj) => { + let blob: &[u8] = obj.extract(py).unwrap(); + let ptr = allocator.new_atom(blob).unwrap(); + ptr + } + PyView::Pair(pair) => { + let pair: &PyTuple = pair.extract(py)?; + + let p0: &PyCell = pair.get_item(0).extract()?; + let p1: &PyCell = pair.get_item(1).extract()?; + let ptr_0 = p0.borrow().check_cache(py, arena)?; + let ptr_1 = p1.borrow().check_cache(py, arena)?; + if let (Some(ptr_0), Some(ptr_1)) = (ptr_0, ptr_1) { + let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); + ptr + } else { + return Ok(Some((p0, p1))); + } + } + } + }; + self.add_to_arena_cache(py, &self.int_arena_cache, new_ptr)?; + } + Ok(None) + } + + pub fn ensure_python_view<'p>( + &'p mut self, + py: Python<'p>, + borrowed_arena: Option<(&PyObject, &mut IntAllocator)>, + ) -> PyResult<&'p PyView> { + // if using `NativeView`, swap it out for `PythonView` + match &self.node { + View::Python(py_view) => Ok(py_view), + View::Native(native_view) => { + let (arena, allocator) = { + if { + if let Some((arena, allocator)) = borrowed_arena { + // WARNING: this check may not actually work + // TODO: research & fix + arena == &native_view.arena + } else { + false + } + } { + borrowed_arena.unwrap() + } else { + let py_int_allocator: PyRef = + native_view.arena.extract(py)?; + let allocator = &mut py_int_allocator.arena; + (&native_view.arena, allocator) + } + }; + self.ensure_py_view_for_ptr_allocator(py, arena, allocator, native_view.ptr) + } + } + } + + fn ensure_py_view_for_ptr_allocator( + &mut self, + py: Python, + arena: &PyObject, + allocator: &mut IntAllocator, + ptr: ::Ptr, + ) -> PyResult<&PyView> { + // create a PyView and put it in self + let py_view = match allocator.sexp(&ptr) { + SExp::Atom(a) => { + let blob = allocator.buf(&a); + let py_bytes = PyBytes::new(py, blob); + PyView::new_atom(py, py_bytes) + } + SExp::Pair(ptr_1, ptr_2) => { + let p1 = Self::from_ptr(py, &arena, ptr_1)?; + let p2 = Self::from_ptr(py, &arena, ptr_2)?; + PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))? + } + }; + self.add_to_arena_cache(py, arena, ptr)?; + let view = View::Python(py_view); + self.node = view; + Ok(&py_view) + } +} + +#[pymethods] +impl PyNaNode { + #[classmethod] + fn new_atom<'p>(cls: &PyType, py: Python<'p>, atom: &PyBytes) -> PyResult<&'p PyCell> { + let node = PyView::new_atom(py, atom); + Self::new(py, View::Python(node)) + } + + #[classmethod] + fn new_pair<'p>( + cls: &PyType, + py: Python<'p>, + p1: &PyCell, + p2: &PyCell, + ) -> PyResult<&'p PyCell> { + let tuple = PyTuple::new(py, &[p1, p2]); + + let node = PyView::new_pair(py, tuple)?; + Self::new(py, View::Python(node)) + } + + #[getter(atom)] + pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult<&'p PyAny> { + let py_view = slf.try_borrow_mut()?.ensure_python_view(py, None)?; + match py_view { + PyView::Atom(obj) => Ok(obj.as_ref(py)), + _ => Ok(py.eval("None", None, None)?.extract()?), + } + } + + #[getter(pair)] + pub fn pair<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult<&'p PyAny> { + let py_view = slf.try_borrow_mut()?.ensure_python_view(py, None)?; + match py_view { + PyView::Pair(obj) => Ok(obj.as_ref(py)), + _ => Ok(py.eval("None", None, None)?.extract()?), + } + } +} diff --git a/src/py/py_view.rs b/src/py/py_view.rs new file mode 100644 index 00000000..5c0a4321 --- /dev/null +++ b/src/py/py_view.rs @@ -0,0 +1,49 @@ +use pyo3::prelude::*; +use pyo3::pycell::PyCell; +use pyo3::types::{PyBytes, PyTuple}; + +use super::py_na_node::PyNaNode; + +pub enum PyView { + Atom(PyObject), + Pair(PyObject), +} + +impl PyView { + pub fn new_atom(py: Python, atom: &PyBytes) -> Self { + PyView::Atom(atom.to_object(py)) + } + + pub fn new_pair(py: Python, pair: &PyTuple) -> PyResult { + if pair.len() != 2 { + py.eval("raise ValueError('new_pair requires 2-tuple')", None, None)?; + } + let p0: &PyCell = pair.get_item(0).extract()?; + let p1: &PyCell = pair.get_item(1).extract()?; + + Ok(PyView::Pair(pair.to_object(py))) + } + + fn atom<'p>(&'p self, py: Python<'p>) -> Option<&'p PyBytes> { + match self { + PyView::Atom(obj) => Some(obj.extract(py).unwrap()), + _ => None, + } + } + + fn pair<'p>(&'p self, py: Python<'p>) -> PyResult> { + match self { + PyView::Pair(obj) => Ok(Some(obj.extract(py)?)), + _ => Ok(None), + } + } + + /* + fn pair_as_cells(&self, py: Python) -> PyResult, &PyCell)>> { + let pair = self.pair(py)?; + let p0: &'p PyCell = pair.get_item(0).extract()?; + let p1: &'p PyCell = pair.get_item(1).extract()?; + Ok((p0, p1)) + } + */ +} From d4c69788e65f0b43c36fdaa054b0c04f7ff69610 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 9 Mar 2021 14:27:28 -0800 Subject: [PATCH 13/76] Checkpoint. Compiles. --- src/py/native_view.rs | 3 ++ src/py/py_int_allocator.rs | 13 ++++++ src/py/py_na_node.rs | 94 ++++++++++++++++++++------------------ src/py/py_view.rs | 1 + 4 files changed, 66 insertions(+), 45 deletions(-) diff --git a/src/py/native_view.rs b/src/py/native_view.rs index 8f73125a..27badcda 100644 --- a/src/py/native_view.rs +++ b/src/py/native_view.rs @@ -4,6 +4,9 @@ use pyo3::types::{PyBytes, PyTuple}; use crate::allocator::Allocator; use crate::int_allocator::IntAllocator; +use super::py_int_allocator::PyIntAllocator; + +#[derive(Clone)] pub struct NativeView { pub arena: PyObject, // PyCell, pub ptr: ::Ptr, diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs index 8b137891..4bd3cda3 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_int_allocator.rs @@ -1 +1,14 @@ +use std::cell::{Cell, Ref, RefCell}; +use pyo3::prelude::*; +use pyo3::types::PyBytes; +use pyo3::types::PyTuple; +use pyo3::types::PyType; + +use crate::allocator::{Allocator, SExp}; +use crate::int_allocator::IntAllocator; + +#[pyclass(subclass, unsendable)] +pub struct PyIntAllocator { + pub arena: IntAllocator, +} diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 6091ece5..2b61c630 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -11,6 +11,7 @@ use super::native_view::NativeView; use super::py_int_allocator::PyIntAllocator; use super::py_view::PyView; +#[derive(Clone)] enum View { Python(PyView), Native(NativeView), @@ -29,14 +30,15 @@ impl PyNaNode { .to_object(py); let node = PyNaNode { - node: view, + node: view.clone(), int_arena_cache, }; - if let View::Native(native_view) = view { - node.add_to_arena_cache(py, &native_view.arena, native_view.ptr)?; - } let r = PyCell::new(py, node)?; + if let View::Native(native_view) = view { + r.borrow() + .add_to_arena_cache(py, &native_view.arena, native_view.ptr)?; + } Ok(r) } @@ -92,36 +94,37 @@ impl PyNaNode { _ => (), } } - Self::ensure_native_view_cached(slf, py, arena, allocator)?; + Self::ensure_native_view_cached(slf.borrow_mut(), py, arena, allocator)?; Ok(slf.try_borrow()?.check_cache(py, arena)?.unwrap()) } - pub fn ensure_native_view_cached( - slf: &PyCell, - py: Python, + pub fn ensure_native_view_cached<'p>( + slf: PyRefMut, + py: Python<'p>, arena: &PyObject, allocator: &mut IntAllocator, ) -> PyResult<()> { - let mut to_cast: Vec<&PyCell> = vec![slf]; + let mut to_cast: Vec> = vec![slf]; Ok(loop { - let t: Option<&PyCell> = to_cast.pop(); + let t: Option> = to_cast.pop(); match t { None => break, - Some(py_cell) => { - let transfer: Option<(&PyCell, &PyCell)> = py_cell - .try_borrow_mut()? - .add_to_native_cache(py, arena, allocator)?; + Some(mut node_ref) => { + let transfer: Option<(&'p PyCell, &'p PyCell)> = + node_ref.add_to_native_cache(py, arena, allocator)?; if let Some((p0, p1)) = transfer { - to_cast.push(p0); - to_cast.push(p1); + to_cast.push(p0.borrow_mut()); + to_cast.push(p1.borrow_mut()); } } } }) } + /// This instance has a corresponding rep in some `IntAllocator` + /// Notate this in the cache. fn add_to_native_cache<'p>( - &'p mut self, + &mut self, py: Python<'p>, arena: &PyObject, allocator: &mut IntAllocator, @@ -139,10 +142,11 @@ impl PyNaNode { ptr } PyView::Pair(pair) => { - let pair: &PyTuple = pair.extract(py)?; + let pair: &'p PyAny = pair.into_ref(py); + let pair: &'p PyTuple = pair.extract()?; - let p0: &PyCell = pair.get_item(0).extract()?; - let p1: &PyCell = pair.get_item(1).extract()?; + let p0: &'p PyCell = pair.get_item(0).extract()?; + let p1: &'p PyCell = pair.get_item(1).extract()?; let ptr_0 = p0.borrow().check_cache(py, arena)?; let ptr_1 = p1.borrow().check_cache(py, arena)?; if let (Some(ptr_0), Some(ptr_1)) = (ptr_0, ptr_1) { @@ -159,34 +163,33 @@ impl PyNaNode { Ok(None) } + /// If this instance is using `NativeView`, replace it with an equivalent `PythonView` + /// so it can be use from python. pub fn ensure_python_view<'p>( - &'p mut self, + &mut self, py: Python<'p>, borrowed_arena: Option<(&PyObject, &mut IntAllocator)>, - ) -> PyResult<&'p PyView> { + ) -> PyResult { // if using `NativeView`, swap it out for `PythonView` - match &self.node { - View::Python(py_view) => Ok(py_view), + match self.node.clone() { + View::Python(py_view) => Ok(py_view.clone()), View::Native(native_view) => { - let (arena, allocator) = { - if { - if let Some((arena, allocator)) = borrowed_arena { - // WARNING: this check may not actually work - // TODO: research & fix - arena == &native_view.arena - } else { - false - } - } { - borrowed_arena.unwrap() - } else { - let py_int_allocator: PyRef = - native_view.arena.extract(py)?; - let allocator = &mut py_int_allocator.arena; - (&native_view.arena, allocator) + let mut py_int_allocator: PyRefMut = native_view.arena.extract(py)?; + let mut allocator_to_use: &mut IntAllocator = &mut py_int_allocator.arena; + + if let Some((arena, allocator)) = borrowed_arena { + // WARNING: this check may not actually work + // TODO: research & fix + if arena == &native_view.arena { + allocator_to_use = allocator; } }; - self.ensure_py_view_for_ptr_allocator(py, arena, allocator, native_view.ptr) + self.ensure_py_view_for_ptr_allocator( + py, + &native_view.arena, + allocator_to_use, + native_view.ptr, + ) } } } @@ -197,7 +200,7 @@ impl PyNaNode { arena: &PyObject, allocator: &mut IntAllocator, ptr: ::Ptr, - ) -> PyResult<&PyView> { + ) -> PyResult { // create a PyView and put it in self let py_view = match allocator.sexp(&ptr) { SExp::Atom(a) => { @@ -212,9 +215,10 @@ impl PyNaNode { } }; self.add_to_arena_cache(py, arena, ptr)?; + let r = py_view.clone(); let view = View::Python(py_view); self.node = view; - Ok(&py_view) + Ok(r) } } @@ -243,7 +247,7 @@ impl PyNaNode { pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult<&'p PyAny> { let py_view = slf.try_borrow_mut()?.ensure_python_view(py, None)?; match py_view { - PyView::Atom(obj) => Ok(obj.as_ref(py)), + PyView::Atom(obj) => Ok(obj.into_ref(py)), _ => Ok(py.eval("None", None, None)?.extract()?), } } @@ -252,7 +256,7 @@ impl PyNaNode { pub fn pair<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult<&'p PyAny> { let py_view = slf.try_borrow_mut()?.ensure_python_view(py, None)?; match py_view { - PyView::Pair(obj) => Ok(obj.as_ref(py)), + PyView::Pair(obj) => Ok(obj.into_ref(py)), _ => Ok(py.eval("None", None, None)?.extract()?), } } diff --git a/src/py/py_view.rs b/src/py/py_view.rs index 5c0a4321..b41e319e 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -4,6 +4,7 @@ use pyo3::types::{PyBytes, PyTuple}; use super::py_na_node::PyNaNode; +#[derive(Clone)] pub enum PyView { Atom(PyObject), Pair(PyObject), From 27d7893f78901bbc32b456a4dd72c1d122e8a0f2 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 9 Mar 2021 21:35:49 -0800 Subject: [PATCH 14/76] Compiles, can create new atom and pair. --- src/py/api.rs | 33 ++++++++++++++++++++------------- src/py/mod.rs | 2 +- src/py/native_view.rs | 5 +---- src/py/op_fn.rs | 2 ++ src/py/py_na_node.rs | 10 ++++++++-- src/py/py_view.rs | 2 +- src/py/run_program.rs | 4 +++- 7 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 1a45a1ee..2aade434 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -6,20 +6,20 @@ use pyo3::wrap_pyfunction; use pyo3::PyObject; use super::glue::{_serialize_from_bytes, _serialize_to_bytes}; -use super::int_allocator_gateway::{PyIntAllocator, PyIntNode}; +//use super::int_allocator_gateway::{PyIntAllocator, PyIntNode}; use super::native_op_lookup::GenericNativeOpLookup; -use super::run_program::{ - __pyo3_get_function_deserialize_and_run_program, __pyo3_get_function_serialized_length, - STRICT_MODE, -}; +use super::py_int_allocator::PyIntAllocator; +use super::py_na_node::PyNaNode; + +//use super::run_program::{__pyo3_get_function_deserialize_and_run_program, STRICT_MODE}; use crate::int_allocator::IntAllocator; use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; -use crate::py::run_program::OperatorHandlerWithMode; +//use crate::py::run_program::OperatorHandlerWithMode; use crate::run_program::OperatorHandler; use crate::{allocator, cost::Cost}; type AllocatorT<'a> = IntAllocator; -type NodeClass = PyIntNode; +//type NodeClass = PyIntNode; /* #[pyclass] @@ -63,6 +63,7 @@ impl NativeOpLookup { } */ +/* #[pyfunction] #[allow(clippy::too_many_arguments)] fn py_run_program( @@ -93,6 +94,7 @@ fn py_run_program( py_callback, ) } +*/ #[pyfunction] fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult { @@ -132,27 +134,31 @@ fn serialize_to_bytes(py: Python, sexp: &PyAny) -> PyResult { /// This module is a python module implemented in Rust. #[pymodule] fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(py_run_program, m)?)?; + //m.add_function(wrap_pyfunction!(py_run_program, m)?)?; //m.add_function(wrap_pyfunction!(serialize_from_bytes, m)?)?; //m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; - m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; - m.add("STRICT_MODE", STRICT_MODE)?; + //m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; + //m.add("STRICT_MODE", STRICT_MODE)?; //m.add_class::()?; // m.add_class::()?; - m.add_class::()?; + //m.add_class::()?; m.add_class::()?; - m.add_function(wrap_pyfunction!(serialized_length, m)?)?; + //m.add_function(wrap_pyfunction!(serialized_length, m)?)?; + m.add_class::()?; + + //m.add_function(wrap_pyfunction!(raise_eval_error, m)?)?; Ok(()) } -use crate::py::op_fn::PyOperatorHandler; +//use crate::py::op_fn::PyOperatorHandler; use crate::reduction::{EvalErr, Reduction}; +/* #[allow(clippy::too_many_arguments)] pub fn _py_run_program( py: Python, @@ -202,3 +208,4 @@ pub fn _py_run_program( } } } +*/ diff --git a/src/py/mod.rs b/src/py/mod.rs index 656fa0a0..6da9ae88 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -2,7 +2,7 @@ pub mod api; pub mod arc_allocator; pub mod f_table; pub mod glue; -pub mod int_allocator_gateway; +//pub mod int_allocator_gateway; pub mod native_op_lookup; pub mod native_view; pub mod op_fn; diff --git a/src/py/native_view.rs b/src/py/native_view.rs index 27badcda..3edab6d7 100644 --- a/src/py/native_view.rs +++ b/src/py/native_view.rs @@ -1,11 +1,8 @@ -use pyo3::prelude::*; -use pyo3::types::{PyBytes, PyTuple}; +use pyo3::prelude::PyObject; use crate::allocator::Allocator; use crate::int_allocator::IntAllocator; -use super::py_int_allocator::PyIntAllocator; - #[derive(Clone)] pub struct NativeView { pub arena: PyObject, // PyCell, diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 1a082817..e2372fc6 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -1,3 +1,4 @@ +/* use std::cell::RefCell; use std::collections::HashMap; @@ -154,3 +155,4 @@ where Ok(o) => Ok(o), } } +*/ diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 2b61c630..16dbdd07 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -26,7 +26,7 @@ pub struct PyNaNode { impl PyNaNode { fn new(py: Python, view: View) -> PyResult<&PyCell> { let int_arena_cache = py - .eval("import weakref; weakref.WeakKeyDictionary()", None, None)? + .eval("__import__('weakref').WeakKeyDictionary()", None, None)? .to_object(py); let node = PyNaNode { @@ -174,7 +174,8 @@ impl PyNaNode { match self.node.clone() { View::Python(py_view) => Ok(py_view.clone()), View::Native(native_view) => { - let mut py_int_allocator: PyRefMut = native_view.arena.extract(py)?; + let mut py_int_allocator: PyRefMut = + native_view.arena.extract(py)?; let mut allocator_to_use: &mut IntAllocator = &mut py_int_allocator.arena; if let Some((arena, allocator)) = borrowed_arena { @@ -260,4 +261,9 @@ impl PyNaNode { _ => Ok(py.eval("None", None, None)?.extract()?), } } + + #[getter(cache)] + fn cache<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { + self.int_arena_cache.extract(py) + } } diff --git a/src/py/py_view.rs b/src/py/py_view.rs index b41e319e..4e9e2145 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -1,4 +1,4 @@ -use pyo3::prelude::*; +use pyo3::{PyObject, PyResult, Python, ToPyObject}; use pyo3::pycell::PyCell; use pyo3::types::{PyBytes, PyTuple}; diff --git a/src/py/run_program.rs b/src/py/run_program.rs index a2562b63..71d1cdb5 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -1,3 +1,4 @@ +/* use std::cell::{Cell, RefCell}; use std::collections::HashMap; @@ -8,7 +9,7 @@ use crate::int_allocator::IntAllocator; use crate::more_ops::op_unknown; use crate::node::Node; use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; -use crate::py::int_allocator_gateway::{PyIntAllocator, PyIntNode}; +use crate::py::py_int_allocator::{PyIntAllocator}; use crate::reduction::Response; use crate::run_program::{run_program, OperatorHandler}; use crate::serialize::{node_from_bytes, node_to_bytes, serialized_length_from_bytes}; @@ -121,3 +122,4 @@ raise EvalError(msg, sexp)", pub fn serialized_length(program: &[u8]) -> PyResult { Ok(serialized_length_from_bytes(program)?) } +*/ From 287aba9a9526f995a03a48df8949ae8faba8e0c0 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 10 Mar 2021 23:34:58 -0800 Subject: [PATCH 15/76] Got `run_program` not completely failing. --- src/py/api.rs | 64 ++++---- src/py/op_fn.rs | 28 ++-- src/py/py_int_allocator.rs | 36 +++++ src/py/py_na_node.rs | 297 +++++++++++++++++++++---------------- src/py/py_view.rs | 2 +- 5 files changed, 259 insertions(+), 168 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 2aade434..169dd56e 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -9,13 +9,14 @@ use super::glue::{_serialize_from_bytes, _serialize_to_bytes}; //use super::int_allocator_gateway::{PyIntAllocator, PyIntNode}; use super::native_op_lookup::GenericNativeOpLookup; use super::py_int_allocator::PyIntAllocator; -use super::py_na_node::PyNaNode; +use super::py_na_node::{new_cache, PyNaNode}; //use super::run_program::{__pyo3_get_function_deserialize_and_run_program, STRICT_MODE}; use crate::int_allocator::IntAllocator; use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; //use crate::py::run_program::OperatorHandlerWithMode; use crate::run_program::OperatorHandler; +use crate::serialize::{node_from_bytes, node_to_bytes}; use crate::{allocator, cost::Cost}; type AllocatorT<'a> = IntAllocator; @@ -116,26 +117,30 @@ const fn allocator_for_py(_py: Python) -> AllocatorT { AllocatorT::new() } -/* #[pyfunction] -fn serialize_from_bytes(py: Python, blob: &[u8]) -> PyIntNode { - let mut allocator = allocator_for_py(py); - - let v = _serialize_from_bytes(&mut allocator, blob); - PyIntNode {} +fn serialize_from_bytes<'p>(py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyCell> { + let mut py_int_allocator = PyCell::new(py, PyIntAllocator::default())?; + let allocator: &mut IntAllocator = &mut py_int_allocator.borrow_mut().arena; + let ptr = node_from_bytes(allocator, blob)?; + PyNaNode::from_ptr(py, &py_int_allocator.to_object(py), ptr) } +/* #[pyfunction] -fn serialize_to_bytes(py: Python, sexp: &PyAny) -> PyResult { - let allocator = allocator_for_py(py); - _serialize_to_bytes::(&allocator, py, sexp) -}*/ +fn serialize_to_bytes(py: Python, sexp: &mut PyNaNode) -> PyResult { + sexp.clear_native_view(); + let mut py_int_allocator: PyIntAllocator = sexp.arena.export(py)?; + let allocator: &mut IntAllocator = &mut py_int_allocator.borrow_mut().arena; + + Ok(node_to_bytes(sexp)) +} +*/ /// This module is a python module implemented in Rust. #[pymodule] fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { - //m.add_function(wrap_pyfunction!(py_run_program, m)?)?; - //m.add_function(wrap_pyfunction!(serialize_from_bytes, m)?)?; + m.add_function(wrap_pyfunction!(py_run_program, m)?)?; + m.add_function(wrap_pyfunction!(serialize_from_bytes, m)?)?; //m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; //m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; @@ -155,30 +160,36 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { Ok(()) } -//use crate::py::op_fn::PyOperatorHandler; +use crate::py::op_fn::PyOperatorHandler; use crate::reduction::{EvalErr, Reduction}; -/* +#[pyfunction] #[allow(clippy::too_many_arguments)] -pub fn _py_run_program( +pub fn py_run_program( py: Python, - program: &PyCell, - args: &PyCell, + program: &PyCell, + args: &PyCell, quote_kw: u8, apply_kw: u8, max_cost: Cost, opcode_lookup_by_name: HashMap>, py_callback: PyObject, ) -> PyResult<(Cost, PyObject)> { - let mut allocator = IntAllocator::new(); - let arena = PyCell::new(py, PyIntAllocator { arena: allocator })?; - let mut b = arena.borrow_mut(); - let mut allocator: &mut IntAllocator = &mut b.arena; - + let cache = new_cache(py)?; + let arena = PyCell::new( + py, + PyIntAllocator { + arena: IntAllocator::new(), + }, + )?; + let mut arena_ref = arena.borrow_mut(); + let mut allocator: &mut IntAllocator = &mut arena_ref.arena; + + let arena_as_obj = arena.to_object(py); println!("1"); - let program = PyIntNode::ptr(program, Some(py), arena.to_object(py), &mut allocator); + let program = PyNaNode::ptr(program, py, &cache, &arena_as_obj, allocator)?; println!("2"); - let args = PyIntNode::ptr(args, Some(py), arena.to_object(py), &mut allocator); + let args = PyNaNode::ptr(args, py, &cache, &arena_as_obj, allocator)?; println!("3"); let op_lookup = Box::new(PyOperatorHandler::new( @@ -193,7 +204,7 @@ pub fn _py_run_program( match r { Ok(reduction) => { - let r = PyIntNode::from_ptr(py, arena.to_object(py), reduction.1)?; + let r = PyNaNode::from_ptr(py, &arena.to_object(py), reduction.1)?; Ok((reduction.0, r.to_object(py))) } Err(eval_err) => { @@ -208,4 +219,3 @@ pub fn _py_run_program( } } } -*/ diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index e2372fc6..3118dd09 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -1,4 +1,3 @@ -/* use std::cell::RefCell; use std::collections::HashMap; @@ -16,12 +15,10 @@ use crate::allocator::Allocator; use crate::cost::Cost; use crate::int_allocator::IntAllocator; use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; -use crate::py::int_allocator_gateway::PyIntNode; +use crate::py::py_na_node::PyNaNode; use crate::reduction::{EvalErr, Reduction, Response}; use crate::run_program::OperatorHandler; -use super::py_node; - pub struct PyOperatorHandler { native_lookup: FLookup, arena: PyObject, @@ -48,6 +45,7 @@ impl PyOperatorHandler { } impl PyOperatorHandler { + /* fn invoke_py_obj( &self, obj: PyObject, @@ -76,10 +74,10 @@ impl PyOperatorHandler { let i0: u32 = unwrap_or_eval_err(pair.get_item(0).extract(), args, "expected u32")?; - let py_node: &PyCell = + let py_node: &PyCell = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let node: i32 = PyIntNode::ptr(py_node, Some(py), arena.clone(), allocator); + let node: i32 = PyNaNode::ptr(py_node, Some(py), arena.clone(), allocator); Ok(Reduction(i0 as Cost, node)) } } @@ -92,16 +90,17 @@ impl PyOperatorHandler { match cache.get(&args) { Some(obj) => obj.clone(), None => { - let py_int_node: &PyCell = - PyCell::new(py, PyIntNode::new(self.arena.clone(), Some(args), None)).unwrap(); + let py_int_node: &PyCell = + PyCell::new(py, PyNaNode::new(self.arena.clone(), Some(args), None)).unwrap(); // this hack ensures we have python representations in all children - PyIntNode::ensure_python_view(vec![py_int_node.to_object(py)], allocator, py); + PyNaNode::ensure_python_view(vec![py_int_node.to_object(py)], allocator, py); let obj: PyObject = py_int_node.to_object(py); cache.insert(args, obj.clone()); obj } } } + */ } impl OperatorHandler for PyOperatorHandler { @@ -119,6 +118,10 @@ impl OperatorHandler for PyOperatorHandler { } } + // HACK TODO remove me + Err(EvalErr(0, "unknown op".to_string())) + + /* self.invoke_py_obj( self.py_callable.clone(), self.arena.clone(), @@ -127,6 +130,7 @@ impl OperatorHandler for PyOperatorHandler { args, max_cost, ) + */ } } @@ -135,13 +139,14 @@ impl OperatorHandler for PyOperatorHandler { fn eval_err_for_pyerr<'p>( py: Python<'p>, pyerr: &PyErr, + cache: PyObject, arena: PyObject, allocator: &mut IntAllocator, ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; - let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = PyIntNode::ptr(&sexp, Some(py), arena, allocator); + let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; + let node: i32 = PyNaNode::ptr(&sexp, py, &cache, &arena, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) } @@ -155,4 +160,3 @@ where Ok(o) => Ok(o), } } -*/ diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs index 4bd3cda3..6ee35353 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_int_allocator.rs @@ -1,9 +1,11 @@ use std::cell::{Cell, Ref, RefCell}; +use pyo3::basic::CompareOp; use pyo3::prelude::*; use pyo3::types::PyBytes; use pyo3::types::PyTuple; use pyo3::types::PyType; +use pyo3::PyObjectProtocol; use crate::allocator::{Allocator, SExp}; use crate::int_allocator::IntAllocator; @@ -12,3 +14,37 @@ use crate::int_allocator::IntAllocator; pub struct PyIntAllocator { pub arena: IntAllocator, } + +impl PyIntAllocator { + fn id(&self) -> isize { + let arena: *const IntAllocator = &self.arena as *const IntAllocator; + arena as isize + } +} + +impl Default for PyIntAllocator { + fn default() -> Self { + PyIntAllocator { + arena: IntAllocator::default(), + } + } +} + +#[pyproto] +impl PyObjectProtocol for PyIntAllocator { + fn __richcmp__(&self, other: PyRef, op: CompareOp) -> i8 { + let t1 = self.id(); + let t2 = other.id(); + if t1 < t2 { + -1 + } else if t2 < t1 { + 1 + } else { + 0 + } + } + + fn __hash__(&self) -> isize { + self.id() + } +} diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 16dbdd07..54bc7384 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -17,45 +17,72 @@ enum View { Native(NativeView), } -#[pyclass] +#[pyclass(weakref)] pub struct PyNaNode { - node: View, - int_arena_cache: PyObject, // WeakKeyDict[PyIntAllocator, int] + py_view: Option, + native_view: Option, + //int_arena_cache: PyObject, // WeakKeyDict[PyIntAllocator, int] } -impl PyNaNode { - fn new(py: Python, view: View) -> PyResult<&PyCell> { - let int_arena_cache = py - .eval("__import__('weakref').WeakKeyDictionary()", None, None)? - .to_object(py); +pub fn new_cache(py: Python) -> PyResult { + Ok(py + .eval("__import__('weakref').WeakValueDictionary()", None, None)? + .to_object(py)) +} - let node = PyNaNode { - node: view.clone(), - int_arena_cache, - }; +fn add_to_cache( + py: Python, + cache: &PyObject, + ptr: ::Ptr, + value: &PyCell, +) -> PyResult<()> { + //return Ok(()); + let locals = [ + ("cache", cache.clone()), + ("key", ptr.to_object(py)), + ("value", value.to_object(py)), + ] + .into_py_dict(py); - let r = PyCell::new(py, node)?; - if let View::Native(native_view) = view { - r.borrow() - .add_to_arena_cache(py, &native_view.arena, native_view.ptr)?; - } - Ok(r) - } + Ok(py.run("cache[key] = value", None, Some(locals))?) +} + +fn from_cache( + py: Python, + cache: &PyObject, + ptr: ::Ptr, +) -> PyResult> { + println!("cc 0"); + let locals = [("cache", cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); + println!("cc 1 {}", locals); + let r = py.eval("cache.get(key)", None, Some(locals))?.extract(); + println!("cc 2"); + r +} - fn add_to_arena_cache( - &self, +impl PyNaNode { + fn new( py: Python, - arena: &PyObject, - ptr: ::Ptr, - ) -> PyResult<()> { - let locals = [ - ("cache", self.int_arena_cache.clone()), - ("key", arena.clone()), - ("value", ptr.to_object(py)), - ] - .into_py_dict(py); + py_view: Option, + native_view: Option, + ) -> PyResult<&PyCell> { + PyCell::new( + py, + PyNaNode { + py_view, + native_view, + }, + ) + } - py.run("cache[key] = value", None, Some(locals))?; + pub fn clear_native_view(&mut self) { + self.native_view = None; + } + + pub fn add_to_cache(slf: &PyCell, py: Python, cache: &PyObject) -> PyResult<()> { + if let Some(native_view) = &slf.borrow().native_view { + add_to_cache(py, cache, native_view.ptr, slf)?; + } Ok(()) } @@ -65,56 +92,53 @@ impl PyNaNode { ptr: ::Ptr, ) -> PyResult<&'p PyCell> { let native_view = NativeView::new(py_int_allocator.clone(), ptr); - Self::new(py, View::Native(native_view)) - } - - fn check_cache( - &self, - py: Python, - key: &PyObject, - ) -> PyResult::Ptr>> { - let locals = [ - ("cache", self.int_arena_cache.clone()), - ("key", key.clone()), - ] - .into_py_dict(py); - py.eval("cache.get(key)", None, Some(locals))?.extract() + Self::new(py, None, Some(native_view)) } pub fn ptr( slf: &PyCell, py: Python, + cache: &PyObject, arena: &PyObject, allocator: &mut IntAllocator, ) -> PyResult<::Ptr> { - { - let self_ref = slf.try_borrow()?; - match self_ref.check_cache(py, arena)? { - Some(v) => return Ok(v), - _ => (), - } + // check if we need to clear the native view + // if arena doesn't match, clear native view + println!("ptr 4"); + Self::populate_native_view(slf, py, cache, arena, allocator)?; + println!("ptr exiting 1"); + if let Some(native_view) = &slf.borrow().native_view { + Ok(native_view.ptr) + } else { + py_raise(py, "oops")? } - Self::ensure_native_view_cached(slf.borrow_mut(), py, arena, allocator)?; - Ok(slf.try_borrow()?.check_cache(py, arena)?.unwrap()) } - pub fn ensure_native_view_cached<'p>( - slf: PyRefMut, + pub fn populate_native_view<'p>( + slf: &PyCell, py: Python<'p>, + cache: &PyObject, arena: &PyObject, allocator: &mut IntAllocator, ) -> PyResult<()> { - let mut to_cast: Vec> = vec![slf]; + let mut to_cast: Vec = vec![slf.to_object(py)]; Ok(loop { - let t: Option> = to_cast.pop(); + println!("pnv vec size {}", to_cast.len()); + let t: Option = to_cast.pop(); match t { None => break, - Some(mut node_ref) => { - let transfer: Option<(&'p PyCell, &'p PyCell)> = - node_ref.add_to_native_cache(py, arena, allocator)?; + Some(node_ref) => { + println!("envc 1"); + let t1: &PyCell = node_ref.extract(py)?; + let transfer: Option<(PyObject, PyObject)> = + Self::add_to_native_cache(t1, py, arena, cache, allocator)?; if let Some((p0, p1)) = transfer { - to_cast.push(p0.borrow_mut()); - to_cast.push(p1.borrow_mut()); + to_cast.push(node_ref); + println!("p0 borrow"); + to_cast.push(p0.to_object(py)); + println!("p1 borrow"); + to_cast.push(p1.to_object(py)); + println!("p1 borrowed"); } } } @@ -124,102 +148,113 @@ impl PyNaNode { /// This instance has a corresponding rep in some `IntAllocator` /// Notate this in the cache. fn add_to_native_cache<'p>( - &mut self, + slf_cell: &PyCell, py: Python<'p>, arena: &PyObject, + cache: &PyObject, allocator: &mut IntAllocator, - ) -> PyResult, &'p PyCell)>> { + ) -> PyResult> { // if it's an atom, we add it to the allocator & cache the addition // if it's a pair, and BOTH are in the cache, we add to allocator & cache // otherwise, we return both children so they can be cached (if necessary) - if self.check_cache(py, arena)?.is_none() { - let py_view = self.ensure_python_view(py, Some((arena, allocator)))?; + let mut slf = slf_cell.borrow_mut(); + let slf: &mut PyNaNode = &mut slf; + if slf.native_view.is_none() { + println!("atnc 1"); + let py_view = slf.populate_python_view(py)?; + println!("atnc 1.5"); let new_ptr = { match py_view { PyView::Atom(obj) => { + println!("atnc 2"); let blob: &[u8] = obj.extract(py).unwrap(); let ptr = allocator.new_atom(blob).unwrap(); + println!("atnc 3 {}", ptr); + add_to_cache(py, cache, ptr, slf_cell); ptr } PyView::Pair(pair) => { - let pair: &'p PyAny = pair.into_ref(py); + println!("atnc 13"); + let pair: &'p PyAny = pair.clone().into_ref(py); let pair: &'p PyTuple = pair.extract()?; + println!("atnc 15"); + let p0: &'p PyCell = pair.get_item(0).extract()?; let p1: &'p PyCell = pair.get_item(1).extract()?; - let ptr_0 = p0.borrow().check_cache(py, arena)?; - let ptr_1 = p1.borrow().check_cache(py, arena)?; + let ptr_0 = match &p0.borrow().native_view { + Some(native_view) => Some(native_view.ptr), + None => None, + }; + let ptr_1 = match &p1.borrow().native_view { + Some(native_view) => Some(native_view.ptr), + None => None, + }; + println!("atnc 17 {:?} {:?}", ptr_0, ptr_1); if let (Some(ptr_0), Some(ptr_1)) = (ptr_0, ptr_1) { let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); + println!("atnc 18 {}", ptr); + add_to_cache(py, cache, ptr, slf_cell); ptr } else { - return Ok(Some((p0, p1))); + println!("atnc 19"); + return Ok(Some((p0.to_object(py), p1.to_object(py)))); } } } }; - self.add_to_arena_cache(py, &self.int_arena_cache, new_ptr)?; + slf.native_view = Some(NativeView::new(arena.clone(), new_ptr)); + Ok(None) + } else { + Ok(None) } - Ok(None) } /// If this instance is using `NativeView`, replace it with an equivalent `PythonView` /// so it can be use from python. - pub fn ensure_python_view<'p>( - &mut self, - py: Python<'p>, - borrowed_arena: Option<(&PyObject, &mut IntAllocator)>, - ) -> PyResult { + pub fn populate_python_view<'p>(&mut self, py: Python<'p>) -> PyResult<&PyView> { // if using `NativeView`, swap it out for `PythonView` - match self.node.clone() { - View::Python(py_view) => Ok(py_view.clone()), - View::Native(native_view) => { - let mut py_int_allocator: PyRefMut = - native_view.arena.extract(py)?; - let mut allocator_to_use: &mut IntAllocator = &mut py_int_allocator.arena; - - if let Some((arena, allocator)) = borrowed_arena { - // WARNING: this check may not actually work - // TODO: research & fix - if arena == &native_view.arena { - allocator_to_use = allocator; - } - }; - self.ensure_py_view_for_ptr_allocator( - py, - &native_view.arena, - allocator_to_use, - native_view.ptr, - ) + println!("ppv 1"); + if self.py_view.is_none() { + println!("ppv 2"); + if let Some(native_view) = &self.native_view { + println!("ppv 3"); + //let mut py_int_allocator: PyRefMut = + // native_view.arena.extract(py)?; + println!("ppv 4"); + //let mut allocator_to_use: &mut IntAllocator = &mut py_int_allocator.arena; + println!("ppv 5"); + self.py_view = Some(Self::py_view_for_native_view(py, native_view)?); + println!("ppv 6"); } } + println!("ppv 40"); + match &self.py_view { + Some(py_view) => return Ok(&py_view), + None => (), + }; + println!("ppv 41"); + py_raise(py, "no pyview available")? } - fn ensure_py_view_for_ptr_allocator( - &mut self, - py: Python, - arena: &PyObject, - allocator: &mut IntAllocator, - ptr: ::Ptr, - ) -> PyResult { - // create a PyView and put it in self - let py_view = match allocator.sexp(&ptr) { + fn py_view_for_native_view(py: Python, native_view: &NativeView) -> PyResult { + let mut py_int_allocator: PyRefMut = native_view.arena.extract(py)?; + let mut allocator: &mut IntAllocator = &mut py_int_allocator.arena; + + // create a PyView and return it + let py_view = match allocator.sexp(&native_view.ptr) { SExp::Atom(a) => { let blob = allocator.buf(&a); let py_bytes = PyBytes::new(py, blob); PyView::new_atom(py, py_bytes) } SExp::Pair(ptr_1, ptr_2) => { - let p1 = Self::from_ptr(py, &arena, ptr_1)?; - let p2 = Self::from_ptr(py, &arena, ptr_2)?; + let p1 = Self::from_ptr(py, &native_view.arena, ptr_1)?; + let p2 = Self::from_ptr(py, &native_view.arena, ptr_2)?; PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))? } }; - self.add_to_arena_cache(py, arena, ptr)?; - let r = py_view.clone(); - let view = View::Python(py_view); - self.node = view; - Ok(r) + Ok(py_view) } } @@ -227,8 +262,8 @@ impl PyNaNode { impl PyNaNode { #[classmethod] fn new_atom<'p>(cls: &PyType, py: Python<'p>, atom: &PyBytes) -> PyResult<&'p PyCell> { - let node = PyView::new_atom(py, atom); - Self::new(py, View::Python(node)) + let py_view = PyView::new_atom(py, atom); + Self::new(py, Some(py_view), None) } #[classmethod] @@ -239,31 +274,37 @@ impl PyNaNode { p2: &PyCell, ) -> PyResult<&'p PyCell> { let tuple = PyTuple::new(py, &[p1, p2]); - - let node = PyView::new_pair(py, tuple)?; - Self::new(py, View::Python(node)) + let py_view = PyView::new_pair(py, tuple)?; + Self::new(py, Some(py_view), None) } #[getter(atom)] - pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult<&'p PyAny> { - let py_view = slf.try_borrow_mut()?.ensure_python_view(py, None)?; + pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { + println!("atom1"); + let mut slf = slf.try_borrow_mut()?; + println!("atom2"); + let py_view: &PyView = slf.populate_python_view(py)?; + println!("atom3"); match py_view { - PyView::Atom(obj) => Ok(obj.into_ref(py)), + PyView::Atom(obj) => Ok(obj.clone()), _ => Ok(py.eval("None", None, None)?.extract()?), } } #[getter(pair)] - pub fn pair<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult<&'p PyAny> { - let py_view = slf.try_borrow_mut()?.ensure_python_view(py, None)?; + pub fn pair<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { + let mut slf = slf.try_borrow_mut()?; + let py_view = slf.populate_python_view(py)?; match py_view { - PyView::Pair(obj) => Ok(obj.into_ref(py)), + PyView::Pair(obj) => Ok(obj.clone()), _ => Ok(py.eval("None", None, None)?.extract()?), } } +} - #[getter(cache)] - fn cache<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { - self.int_arena_cache.extract(py) - } +fn py_raise(py: Python, msg: &str) -> PyResult { + let locals = [("msg", msg.to_object(py))].into_py_dict(py); + + py.run("raise RuntimeError(msg)", None, Some(locals))?; + panic!("we should never get here") } diff --git a/src/py/py_view.rs b/src/py/py_view.rs index 4e9e2145..1dcc6ea3 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -1,6 +1,6 @@ -use pyo3::{PyObject, PyResult, Python, ToPyObject}; use pyo3::pycell::PyCell; use pyo3::types::{PyBytes, PyTuple}; +use pyo3::{PyObject, PyResult, Python, ToPyObject}; use super::py_na_node::PyNaNode; From 5b2c669bc822dfb65599e6977f01e73b726983a1 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 12 Mar 2021 15:22:48 -0800 Subject: [PATCH 16/76] Maybe works now? --- src/py/api.rs | 39 ++++++++++++++++++++++++++-------- src/py/py_na_node.rs | 50 +++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 169dd56e..94a576a2 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use pyo3::prelude::*; -use pyo3::types::{PyDict, PyString}; +use pyo3::types::{PyBytes, PyDict, PyString}; use pyo3::wrap_pyfunction; use pyo3::PyObject; @@ -125,23 +125,43 @@ fn serialize_from_bytes<'p>(py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyCell< PyNaNode::from_ptr(py, &py_int_allocator.to_object(py), ptr) } -/* +use crate::node::Node; + #[pyfunction] -fn serialize_to_bytes(py: Python, sexp: &mut PyNaNode) -> PyResult { - sexp.clear_native_view(); - let mut py_int_allocator: PyIntAllocator = sexp.arena.export(py)?; - let allocator: &mut IntAllocator = &mut py_int_allocator.borrow_mut().arena; +fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { + println!("s2b 1"); + PyNaNode::clear_native_view(sexp, py); + + println!("s2b 2"); + let mut py_int_allocator_cell = PyCell::new(py, PyIntAllocator::default())?; + println!("s2b 3"); + let mut py_int_allocator: &mut PyIntAllocator = &mut py_int_allocator_cell.borrow_mut(); + println!("s2b 4"); + let allocator: &mut IntAllocator = &mut py_int_allocator.arena; + println!("s2b 5"); + + let ptr = PyNaNode::ptr( + sexp, + py, + &new_cache(py)?, + &py_int_allocator_cell.to_object(py), + allocator, + )?; + println!("s2b 6"); - Ok(node_to_bytes(sexp)) + let node = Node::new(allocator, ptr); + println!("s2b 7"); + let s: Vec = node_to_bytes(&node)?; + println!("s2b 8"); + Ok(PyBytes::new(py, &s)) } -*/ /// This module is a python module implemented in Rust. #[pymodule] fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(py_run_program, m)?)?; m.add_function(wrap_pyfunction!(serialize_from_bytes, m)?)?; - //m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; + m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; //m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; //m.add("STRICT_MODE", STRICT_MODE)?; @@ -187,6 +207,7 @@ pub fn py_run_program( let arena_as_obj = arena.to_object(py); println!("1"); + PyNaNode::clear_native_view(program, py); let program = PyNaNode::ptr(program, py, &cache, &arena_as_obj, allocator)?; println!("2"); let args = PyNaNode::ptr(args, py, &cache, &arena_as_obj, allocator)?; diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 54bc7384..4181856d 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -17,7 +17,7 @@ enum View { Native(NativeView), } -#[pyclass(weakref)] +#[pyclass(weakref, subclass)] pub struct PyNaNode { py_view: Option, native_view: Option, @@ -75,8 +75,26 @@ impl PyNaNode { ) } - pub fn clear_native_view(&mut self) { - self.native_view = None; + pub fn clear_native_view(slf: &PyCell, py: Python) -> PyResult<()> { + let mut items = vec![slf.to_object(py)]; + loop { + let t = items.pop(); + if let Some(obj) = t { + let mut node: PyRefMut = obj.extract(py)?; + node.populate_python_view(py)?; + assert!(node.py_view.is_some()); + node.native_view = None; + if let Some(PyView::Pair(tuple)) = &node.py_view { + let (p0, p1): (PyObject, PyObject) = tuple.extract(py)?; + //let (p0, p1): (&PyCell, &PyCell) = tuple.extract(py)?; + items.push(p0); + items.push(p1); + } + } else { + break; + } + } + Ok(()) } pub fn add_to_cache(slf: &PyCell, py: Python, cache: &PyObject) -> PyResult<()> { @@ -226,6 +244,8 @@ impl PyNaNode { println!("ppv 5"); self.py_view = Some(Self::py_view_for_native_view(py, native_view)?); println!("ppv 6"); + } else { + panic!("missing native AND python view"); } } println!("ppv 40"); @@ -260,6 +280,24 @@ impl PyNaNode { #[pymethods] impl PyNaNode { + #[new] + fn new_obj<'p>(py: Python<'p>, obj: &PyAny) -> PyResult { + Ok(if let Ok(tuple) = obj.extract() { + let py_view = PyView::new_pair(py, tuple)?; + Self { + py_view: Some(py_view), + native_view: None, + } + } else { + let py_bytes: &PyBytes = obj.extract()?; + let py_view = PyView::new_atom(py, py_bytes); + Self { + py_view: Some(py_view), + native_view: None, + } + }) + } + #[classmethod] fn new_atom<'p>(cls: &PyType, py: Python<'p>, atom: &PyBytes) -> PyResult<&'p PyCell> { let py_view = PyView::new_atom(py, atom); @@ -278,6 +316,12 @@ impl PyNaNode { Self::new(py, Some(py_view), None) } + #[classmethod] + fn new_tuple<'p>(cls: &PyType, py: Python<'p>, tuple: &PyTuple) -> PyResult<&'p PyCell> { + let py_view = PyView::new_pair(py, tuple)?; + Self::new(py, Some(py_view), None) + } + #[getter(atom)] pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { println!("atom1"); From cda51664cee1aac3045da4dd220419f6913f6429 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 12 Mar 2021 16:47:41 -0800 Subject: [PATCH 17/76] More in `PyNaNode`. --- src/py/py_na_node.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 4181856d..e06dfce7 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -344,6 +344,46 @@ impl PyNaNode { _ => Ok(py.eval("None", None, None)?.extract()?), } } + + #[getter(native)] + pub fn native<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult<&'p PyAny> { + match &slf.borrow().native_view { + Some(native_view) => { + let locals = [ + ("a", native_view.arena.clone()), + ("b", native_view.ptr.into_py(py)), + ] + .into_py_dict(py); + py.eval("(a, b)", None, Some(locals)) + } + None => py.eval("None", None, None), + } + } + + #[getter(python)] + pub fn python<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { + Ok(match &slf.borrow().py_view { + Some(PyView::Atom(atom)) => ("Atom", atom).to_object(py), + Some(PyView::Pair(pair)) => ("Pair", pair).to_object(py), + _ => py.eval("None", None, None)?.to_object(py), + }) + } + + pub fn ensure_python(slf: &PyCell, py: Python) -> PyResult<()> { + Self::populate_python_view(&mut slf.borrow_mut(), py)?; + Ok(()) + } + + pub fn ensure_native( + slf: &PyCell, + py: Python, + cache: PyObject, + arena: PyObject, + ) -> PyResult<()> { + let py_int_allocator: &PyCell = arena.extract(py)?; + let mut allocator: &mut IntAllocator = &mut py_int_allocator.borrow_mut().arena; + Self::populate_native_view(slf, py, &cache, &arena, allocator) + } } fn py_raise(py: Python, msg: &str) -> PyResult { From 98ccf3e475ab28e1edfec1f081bc559a82428854 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 12 Mar 2021 16:50:49 -0800 Subject: [PATCH 18/76] Remove `println!` debug messages. --- src/py/api.rs | 11 ----------- src/py/py_na_node.rs | 35 +---------------------------------- src/tests.rs | 2 -- 3 files changed, 1 insertion(+), 47 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 94a576a2..d93896f6 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -129,16 +129,11 @@ use crate::node::Node; #[pyfunction] fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { - println!("s2b 1"); PyNaNode::clear_native_view(sexp, py); - println!("s2b 2"); let mut py_int_allocator_cell = PyCell::new(py, PyIntAllocator::default())?; - println!("s2b 3"); let mut py_int_allocator: &mut PyIntAllocator = &mut py_int_allocator_cell.borrow_mut(); - println!("s2b 4"); let allocator: &mut IntAllocator = &mut py_int_allocator.arena; - println!("s2b 5"); let ptr = PyNaNode::ptr( sexp, @@ -147,12 +142,9 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<& &py_int_allocator_cell.to_object(py), allocator, )?; - println!("s2b 6"); let node = Node::new(allocator, ptr); - println!("s2b 7"); let s: Vec = node_to_bytes(&node)?; - println!("s2b 8"); Ok(PyBytes::new(py, &s)) } @@ -206,12 +198,9 @@ pub fn py_run_program( let mut allocator: &mut IntAllocator = &mut arena_ref.arena; let arena_as_obj = arena.to_object(py); - println!("1"); PyNaNode::clear_native_view(program, py); let program = PyNaNode::ptr(program, py, &cache, &arena_as_obj, allocator)?; - println!("2"); let args = PyNaNode::ptr(args, py, &cache, &arena_as_obj, allocator)?; - println!("3"); let op_lookup = Box::new(PyOperatorHandler::new( opcode_lookup_by_name, diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index e06dfce7..911b2379 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -52,12 +52,8 @@ fn from_cache( cache: &PyObject, ptr: ::Ptr, ) -> PyResult> { - println!("cc 0"); let locals = [("cache", cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); - println!("cc 1 {}", locals); - let r = py.eval("cache.get(key)", None, Some(locals))?.extract(); - println!("cc 2"); - r + py.eval("cache.get(key)", None, Some(locals))?.extract() } impl PyNaNode { @@ -122,9 +118,7 @@ impl PyNaNode { ) -> PyResult<::Ptr> { // check if we need to clear the native view // if arena doesn't match, clear native view - println!("ptr 4"); Self::populate_native_view(slf, py, cache, arena, allocator)?; - println!("ptr exiting 1"); if let Some(native_view) = &slf.borrow().native_view { Ok(native_view.ptr) } else { @@ -141,22 +135,17 @@ impl PyNaNode { ) -> PyResult<()> { let mut to_cast: Vec = vec![slf.to_object(py)]; Ok(loop { - println!("pnv vec size {}", to_cast.len()); let t: Option = to_cast.pop(); match t { None => break, Some(node_ref) => { - println!("envc 1"); let t1: &PyCell = node_ref.extract(py)?; let transfer: Option<(PyObject, PyObject)> = Self::add_to_native_cache(t1, py, arena, cache, allocator)?; if let Some((p0, p1)) = transfer { to_cast.push(node_ref); - println!("p0 borrow"); to_cast.push(p0.to_object(py)); - println!("p1 borrow"); to_cast.push(p1.to_object(py)); - println!("p1 borrowed"); } } } @@ -178,26 +167,18 @@ impl PyNaNode { let mut slf = slf_cell.borrow_mut(); let slf: &mut PyNaNode = &mut slf; if slf.native_view.is_none() { - println!("atnc 1"); let py_view = slf.populate_python_view(py)?; - println!("atnc 1.5"); let new_ptr = { match py_view { PyView::Atom(obj) => { - println!("atnc 2"); let blob: &[u8] = obj.extract(py).unwrap(); let ptr = allocator.new_atom(blob).unwrap(); - println!("atnc 3 {}", ptr); add_to_cache(py, cache, ptr, slf_cell); ptr } PyView::Pair(pair) => { - println!("atnc 13"); let pair: &'p PyAny = pair.clone().into_ref(py); let pair: &'p PyTuple = pair.extract()?; - - println!("atnc 15"); - let p0: &'p PyCell = pair.get_item(0).extract()?; let p1: &'p PyCell = pair.get_item(1).extract()?; let ptr_0 = match &p0.borrow().native_view { @@ -208,14 +189,11 @@ impl PyNaNode { Some(native_view) => Some(native_view.ptr), None => None, }; - println!("atnc 17 {:?} {:?}", ptr_0, ptr_1); if let (Some(ptr_0), Some(ptr_1)) = (ptr_0, ptr_1) { let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); - println!("atnc 18 {}", ptr); add_to_cache(py, cache, ptr, slf_cell); ptr } else { - println!("atnc 19"); return Ok(Some((p0.to_object(py), p1.to_object(py)))); } } @@ -232,28 +210,20 @@ impl PyNaNode { /// so it can be use from python. pub fn populate_python_view<'p>(&mut self, py: Python<'p>) -> PyResult<&PyView> { // if using `NativeView`, swap it out for `PythonView` - println!("ppv 1"); if self.py_view.is_none() { - println!("ppv 2"); if let Some(native_view) = &self.native_view { - println!("ppv 3"); //let mut py_int_allocator: PyRefMut = // native_view.arena.extract(py)?; - println!("ppv 4"); //let mut allocator_to_use: &mut IntAllocator = &mut py_int_allocator.arena; - println!("ppv 5"); self.py_view = Some(Self::py_view_for_native_view(py, native_view)?); - println!("ppv 6"); } else { panic!("missing native AND python view"); } } - println!("ppv 40"); match &self.py_view { Some(py_view) => return Ok(&py_view), None => (), }; - println!("ppv 41"); py_raise(py, "no pyview available")? } @@ -324,11 +294,8 @@ impl PyNaNode { #[getter(atom)] pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - println!("atom1"); let mut slf = slf.try_borrow_mut()?; - println!("atom2"); let py_view: &PyView = slf.populate_python_view(py)?; - println!("atom3"); match py_view { PyView::Atom(obj) => Ok(obj.clone()), _ => Ok(py.eval("None", None, None)?.extract()?), diff --git a/src/tests.rs b/src/tests.rs index b007252f..4c7f3b02 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -161,9 +161,7 @@ fn node_to_hex(node: &Node) -> String { fn do_test_run_program(input_as_hex: &str, expected_as_hex: &str) -> () { let mut a = IntAllocator::new(); let n = node_from_hex(&a, input_as_hex); - println!("n = {:?}", n); let r = do_run_program(&n, &null); - println!("r = {:?}", r); assert_eq!(node_to_hex(&r), expected_as_hex); } From 864723344d0aa1f01e771658cc1452971efe42e1 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Sat, 13 Mar 2021 16:48:20 -0800 Subject: [PATCH 19/76] Run `cargo fix`. --- src/py/api.rs | 8 ++++---- src/py/op_fn.rs | 4 ++-- src/py/py_int_allocator.rs | 12 ++++++------ src/py/py_na_node.rs | 12 ++++++------ src/py/py_view.rs | 4 ++-- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index d93896f6..2c61895c 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -119,7 +119,7 @@ const fn allocator_for_py(_py: Python) -> AllocatorT { #[pyfunction] fn serialize_from_bytes<'p>(py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyCell> { - let mut py_int_allocator = PyCell::new(py, PyIntAllocator::default())?; + let py_int_allocator = PyCell::new(py, PyIntAllocator::default())?; let allocator: &mut IntAllocator = &mut py_int_allocator.borrow_mut().arena; let ptr = node_from_bytes(allocator, blob)?; PyNaNode::from_ptr(py, &py_int_allocator.to_object(py), ptr) @@ -131,8 +131,8 @@ use crate::node::Node; fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { PyNaNode::clear_native_view(sexp, py); - let mut py_int_allocator_cell = PyCell::new(py, PyIntAllocator::default())?; - let mut py_int_allocator: &mut PyIntAllocator = &mut py_int_allocator_cell.borrow_mut(); + let py_int_allocator_cell = PyCell::new(py, PyIntAllocator::default())?; + let py_int_allocator: &mut PyIntAllocator = &mut py_int_allocator_cell.borrow_mut(); let allocator: &mut IntAllocator = &mut py_int_allocator.arena; let ptr = PyNaNode::ptr( @@ -195,7 +195,7 @@ pub fn py_run_program( }, )?; let mut arena_ref = arena.borrow_mut(); - let mut allocator: &mut IntAllocator = &mut arena_ref.arena; + let allocator: &mut IntAllocator = &mut arena_ref.arena; let arena_as_obj = arena.to_object(py); PyNaNode::clear_native_view(program, py); diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 3118dd09..9e87701c 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -9,14 +9,14 @@ use pyo3::PyErr; use pyo3::PyObject; use pyo3::PyResult; use pyo3::Python; -use pyo3::ToPyObject; + use crate::allocator::Allocator; use crate::cost::Cost; use crate::int_allocator::IntAllocator; use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; use crate::py::py_na_node::PyNaNode; -use crate::reduction::{EvalErr, Reduction, Response}; +use crate::reduction::{EvalErr, Response}; use crate::run_program::OperatorHandler; pub struct PyOperatorHandler { diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs index 6ee35353..8cabdec9 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_int_allocator.rs @@ -1,13 +1,13 @@ -use std::cell::{Cell, Ref, RefCell}; + use pyo3::basic::CompareOp; use pyo3::prelude::*; -use pyo3::types::PyBytes; -use pyo3::types::PyTuple; -use pyo3::types::PyType; + + + use pyo3::PyObjectProtocol; -use crate::allocator::{Allocator, SExp}; + use crate::int_allocator::IntAllocator; #[pyclass(subclass, unsendable)] @@ -32,7 +32,7 @@ impl Default for PyIntAllocator { #[pyproto] impl PyObjectProtocol for PyIntAllocator { - fn __richcmp__(&self, other: PyRef, op: CompareOp) -> i8 { + fn __richcmp__(&self, other: PyRef, _op: CompareOp) -> i8 { let t1 = self.id(); let t2 = other.id(); if t1 < t2 { diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 911b2379..509c241c 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -1,5 +1,5 @@ use std::borrow::Borrow; -use std::cell::{Ref, RefCell}; + use pyo3::prelude::*; use pyo3::types::{IntoPyDict, PyBytes, PyTuple, PyType}; @@ -229,7 +229,7 @@ impl PyNaNode { fn py_view_for_native_view(py: Python, native_view: &NativeView) -> PyResult { let mut py_int_allocator: PyRefMut = native_view.arena.extract(py)?; - let mut allocator: &mut IntAllocator = &mut py_int_allocator.arena; + let allocator: &mut IntAllocator = &mut py_int_allocator.arena; // create a PyView and return it let py_view = match allocator.sexp(&native_view.ptr) { @@ -269,14 +269,14 @@ impl PyNaNode { } #[classmethod] - fn new_atom<'p>(cls: &PyType, py: Python<'p>, atom: &PyBytes) -> PyResult<&'p PyCell> { + fn new_atom<'p>(_cls: &PyType, py: Python<'p>, atom: &PyBytes) -> PyResult<&'p PyCell> { let py_view = PyView::new_atom(py, atom); Self::new(py, Some(py_view), None) } #[classmethod] fn new_pair<'p>( - cls: &PyType, + _cls: &PyType, py: Python<'p>, p1: &PyCell, p2: &PyCell, @@ -287,7 +287,7 @@ impl PyNaNode { } #[classmethod] - fn new_tuple<'p>(cls: &PyType, py: Python<'p>, tuple: &PyTuple) -> PyResult<&'p PyCell> { + fn new_tuple<'p>(_cls: &PyType, py: Python<'p>, tuple: &PyTuple) -> PyResult<&'p PyCell> { let py_view = PyView::new_pair(py, tuple)?; Self::new(py, Some(py_view), None) } @@ -348,7 +348,7 @@ impl PyNaNode { arena: PyObject, ) -> PyResult<()> { let py_int_allocator: &PyCell = arena.extract(py)?; - let mut allocator: &mut IntAllocator = &mut py_int_allocator.borrow_mut().arena; + let allocator: &mut IntAllocator = &mut py_int_allocator.borrow_mut().arena; Self::populate_native_view(slf, py, &cache, &arena, allocator) } } diff --git a/src/py/py_view.rs b/src/py/py_view.rs index 1dcc6ea3..ffd8f8f9 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -19,8 +19,8 @@ impl PyView { if pair.len() != 2 { py.eval("raise ValueError('new_pair requires 2-tuple')", None, None)?; } - let p0: &PyCell = pair.get_item(0).extract()?; - let p1: &PyCell = pair.get_item(1).extract()?; + let _p0: &PyCell = pair.get_item(0).extract()?; + let _p1: &PyCell = pair.get_item(1).extract()?; Ok(PyView::Pair(pair.to_object(py))) } From 40848e67228ad0bce22250314507f1d4d2c8aa7c Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Sat, 13 Mar 2021 16:55:06 -0800 Subject: [PATCH 20/76] Do `cargo clippy`. --- src/py/api.rs | 13 ++++++------- src/py/op_fn.rs | 4 +--- src/py/py_int_allocator.rs | 31 ------------------------------- src/py/py_na_node.rs | 20 ++++++-------------- src/py/py_view.rs | 23 ----------------------- 5 files changed, 13 insertions(+), 78 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 2c61895c..589ed786 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -5,19 +5,18 @@ use pyo3::types::{PyBytes, PyDict, PyString}; use pyo3::wrap_pyfunction; use pyo3::PyObject; -use super::glue::{_serialize_from_bytes, _serialize_to_bytes}; //use super::int_allocator_gateway::{PyIntAllocator, PyIntNode}; -use super::native_op_lookup::GenericNativeOpLookup; + use super::py_int_allocator::PyIntAllocator; use super::py_na_node::{new_cache, PyNaNode}; //use super::run_program::{__pyo3_get_function_deserialize_and_run_program, STRICT_MODE}; use crate::int_allocator::IntAllocator; -use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; + //use crate::py::run_program::OperatorHandlerWithMode; -use crate::run_program::OperatorHandler; + +use crate::cost::Cost; use crate::serialize::{node_from_bytes, node_to_bytes}; -use crate::{allocator, cost::Cost}; type AllocatorT<'a> = IntAllocator; //type NodeClass = PyIntNode; @@ -129,7 +128,7 @@ use crate::node::Node; #[pyfunction] fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { - PyNaNode::clear_native_view(sexp, py); + PyNaNode::clear_native_view(sexp, py)?; let py_int_allocator_cell = PyCell::new(py, PyIntAllocator::default())?; let py_int_allocator: &mut PyIntAllocator = &mut py_int_allocator_cell.borrow_mut(); @@ -198,7 +197,7 @@ pub fn py_run_program( let allocator: &mut IntAllocator = &mut arena_ref.arena; let arena_as_obj = arena.to_object(py); - PyNaNode::clear_native_view(program, py); + PyNaNode::clear_native_view(program, py)?; let program = PyNaNode::ptr(program, py, &cache, &arena_as_obj, allocator)?; let args = PyNaNode::ptr(args, py, &cache, &arena_as_obj, allocator)?; diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 9e87701c..4665c655 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -1,7 +1,6 @@ use std::cell::RefCell; use std::collections::HashMap; -use pyo3::prelude::pyclass; use pyo3::types::PyString; use pyo3::types::PyTuple; use pyo3::PyCell; @@ -10,7 +9,6 @@ use pyo3::PyObject; use pyo3::PyResult; use pyo3::Python; - use crate::allocator::Allocator; use crate::cost::Cost; use crate::int_allocator::IntAllocator; @@ -114,7 +112,7 @@ impl OperatorHandler for PyOperatorHandler { let op = allocator.buf(&op_buf); if op.len() == 1 { if let Some(f) = self.native_lookup[op[0] as usize] { - return f(allocator, args.clone(), max_cost); + return f(allocator, *args, max_cost); } } diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs index 8cabdec9..80a87c76 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_int_allocator.rs @@ -1,13 +1,8 @@ - - use pyo3::basic::CompareOp; use pyo3::prelude::*; - - use pyo3::PyObjectProtocol; - use crate::int_allocator::IntAllocator; #[pyclass(subclass, unsendable)] @@ -15,13 +10,6 @@ pub struct PyIntAllocator { pub arena: IntAllocator, } -impl PyIntAllocator { - fn id(&self) -> isize { - let arena: *const IntAllocator = &self.arena as *const IntAllocator; - arena as isize - } -} - impl Default for PyIntAllocator { fn default() -> Self { PyIntAllocator { @@ -29,22 +17,3 @@ impl Default for PyIntAllocator { } } } - -#[pyproto] -impl PyObjectProtocol for PyIntAllocator { - fn __richcmp__(&self, other: PyRef, _op: CompareOp) -> i8 { - let t1 = self.id(); - let t2 = other.id(); - if t1 < t2 { - -1 - } else if t2 < t1 { - 1 - } else { - 0 - } - } - - fn __hash__(&self) -> isize { - self.id() - } -} diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 509c241c..ac23f354 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -1,6 +1,3 @@ -use std::borrow::Borrow; - - use pyo3::prelude::*; use pyo3::types::{IntoPyDict, PyBytes, PyTuple, PyType}; @@ -11,12 +8,6 @@ use super::native_view::NativeView; use super::py_int_allocator::PyIntAllocator; use super::py_view::PyView; -#[derive(Clone)] -enum View { - Python(PyView), - Native(NativeView), -} - #[pyclass(weakref, subclass)] pub struct PyNaNode { py_view: Option, @@ -134,7 +125,7 @@ impl PyNaNode { allocator: &mut IntAllocator, ) -> PyResult<()> { let mut to_cast: Vec = vec![slf.to_object(py)]; - Ok(loop { + loop { let t: Option = to_cast.pop(); match t { None => break, @@ -149,7 +140,8 @@ impl PyNaNode { } } } - }) + } + Ok(()) } /// This instance has a corresponding rep in some `IntAllocator` @@ -173,7 +165,7 @@ impl PyNaNode { PyView::Atom(obj) => { let blob: &[u8] = obj.extract(py).unwrap(); let ptr = allocator.new_atom(blob).unwrap(); - add_to_cache(py, cache, ptr, slf_cell); + add_to_cache(py, cache, ptr, slf_cell)?; ptr } PyView::Pair(pair) => { @@ -191,7 +183,7 @@ impl PyNaNode { }; if let (Some(ptr_0), Some(ptr_1)) = (ptr_0, ptr_1) { let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); - add_to_cache(py, cache, ptr, slf_cell); + add_to_cache(py, cache, ptr, slf_cell)?; ptr } else { return Ok(Some((p0.to_object(py), p1.to_object(py)))); @@ -251,7 +243,7 @@ impl PyNaNode { #[pymethods] impl PyNaNode { #[new] - fn new_obj<'p>(py: Python<'p>, obj: &PyAny) -> PyResult { + fn new_obj(py: Python, obj: &PyAny) -> PyResult { Ok(if let Ok(tuple) = obj.extract() { let py_view = PyView::new_pair(py, tuple)?; Self { diff --git a/src/py/py_view.rs b/src/py/py_view.rs index ffd8f8f9..16c78143 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -24,27 +24,4 @@ impl PyView { Ok(PyView::Pair(pair.to_object(py))) } - - fn atom<'p>(&'p self, py: Python<'p>) -> Option<&'p PyBytes> { - match self { - PyView::Atom(obj) => Some(obj.extract(py).unwrap()), - _ => None, - } - } - - fn pair<'p>(&'p self, py: Python<'p>) -> PyResult> { - match self { - PyView::Pair(obj) => Ok(Some(obj.extract(py)?)), - _ => Ok(None), - } - } - - /* - fn pair_as_cells(&self, py: Python) -> PyResult, &PyCell)>> { - let pair = self.pair(py)?; - let p0: &'p PyCell = pair.get_item(0).extract()?; - let p1: &'p PyCell = pair.get_item(1).extract()?; - Ok((p0, p1)) - } - */ } From b5dbd5a325e8b55aeffddc537083dfaf4bc2e162 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Sat, 13 Mar 2021 17:32:52 -0800 Subject: [PATCH 21/76] Faster `None`. --- src/py/int_allocator_gateway.rs | 4 ++-- src/py/py_na_node.rs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs index 006f6ec2..79b0badb 100644 --- a/src/py/int_allocator_gateway.rs +++ b/src/py/int_allocator_gateway.rs @@ -238,7 +238,7 @@ impl PyIntNode { impl PyIntNode { #[classmethod] fn new_atom(cls: &PyType, py: Python, atom: &PyBytes) -> PyResult { - let none: PyObject = py.eval("None", None, None)?.to_object(py); + let none: PyObject = py.None(); let py_view = Some(PyView::new(&atom.to_object(py), &none)); Ok(PyIntNode::new(none, None, py_view)) } @@ -251,7 +251,7 @@ impl PyIntNode { p2: &PyCell, ) -> PyResult { let pair: &PyTuple = PyTuple::new(py, &[p1, p2]); - let none: PyObject = py.eval("None", None, None)?.to_object(py); + let none: PyObject = py.None(); // TODO: ensure `pair` is a tuple of two `PyIntNode` let py_view = Some(PyView::new(&none, &pair.to_object(py))); Ok(PyIntNode::new(none, None, py_view)) diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index ac23f354..3d94e6a4 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -290,7 +290,7 @@ impl PyNaNode { let py_view: &PyView = slf.populate_python_view(py)?; match py_view { PyView::Atom(obj) => Ok(obj.clone()), - _ => Ok(py.eval("None", None, None)?.extract()?), + _ => Ok(py.None()), } } @@ -300,23 +300,23 @@ impl PyNaNode { let py_view = slf.populate_python_view(py)?; match py_view { PyView::Pair(obj) => Ok(obj.clone()), - _ => Ok(py.eval("None", None, None)?.extract()?), + _ => Ok(py.None()), } } #[getter(native)] - pub fn native<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult<&'p PyAny> { - match &slf.borrow().native_view { + pub fn native<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { + Ok(match &slf.borrow().native_view { Some(native_view) => { let locals = [ ("a", native_view.arena.clone()), ("b", native_view.ptr.into_py(py)), ] .into_py_dict(py); - py.eval("(a, b)", None, Some(locals)) + py.eval("(a, b)", None, Some(locals))?.to_object(py) } - None => py.eval("None", None, None), - } + None => py.None(), + }) } #[getter(python)] @@ -324,7 +324,7 @@ impl PyNaNode { Ok(match &slf.borrow().py_view { Some(PyView::Atom(atom)) => ("Atom", atom).to_object(py), Some(PyView::Pair(pair)) => ("Pair", pair).to_object(py), - _ => py.eval("None", None, None)?.to_object(py), + _ => py.None(), }) } From e6e0d4127dfcb957e0275547389416c89032f55e Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Sat, 13 Mar 2021 18:43:07 -0800 Subject: [PATCH 22/76] Checkpoint. Almost have py operators working. --- src/py/api.rs | 1 + src/py/op_fn.rs | 57 ++++++++++++++++++++------------------------ src/py/py_na_node.rs | 4 ++-- 3 files changed, 29 insertions(+), 33 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 589ed786..1455fc22 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -204,6 +204,7 @@ pub fn py_run_program( let op_lookup = Box::new(PyOperatorHandler::new( opcode_lookup_by_name, arena.to_object(py), + cache, py_callback, )); diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 4665c655..da69ea6b 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -8,31 +8,32 @@ use pyo3::PyErr; use pyo3::PyObject; use pyo3::PyResult; use pyo3::Python; +use pyo3::ToPyObject; use crate::allocator::Allocator; use crate::cost::Cost; use crate::int_allocator::IntAllocator; use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; -use crate::py::py_na_node::PyNaNode; -use crate::reduction::{EvalErr, Response}; +use crate::py::py_na_node::{add_to_cache, from_cache, PyNaNode}; +use crate::reduction::{EvalErr, Reduction, Response}; use crate::run_program::OperatorHandler; pub struct PyOperatorHandler { native_lookup: FLookup, arena: PyObject, py_callable: PyObject, - cache: RefCell>, + cache: PyObject, } impl PyOperatorHandler { pub fn new( opcode_lookup_by_name: HashMap>, arena: PyObject, + cache: PyObject, py_callable: PyObject, ) -> Self { let native_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); - let cache = RefCell::new(HashMap::new()); PyOperatorHandler { native_lookup, arena, @@ -43,8 +44,7 @@ impl PyOperatorHandler { } impl PyOperatorHandler { - /* - fn invoke_py_obj( + pub fn invoke_py_obj( &self, obj: PyObject, arena: PyObject, @@ -54,14 +54,23 @@ impl PyOperatorHandler { max_cost: Cost, ) -> Response<::Ptr> { Python::with_gil(|py| { - let op = allocator.buf(&op_buf).to_object(py); - let py_int_node = self.uncache(py, allocator, args); + let op: &[u8] = allocator.buf(&op_buf); + let r = self.uncache(py, args); + let py_int_node = unwrap_or_eval_err(r, args, "can't uncache")?; + // TODO: implement a `populate_python_view` that accepts the borrowed `allocator` above + // since the existing one will try to re-borrow it and fail + // py_int_node.populate_python_view(py); let r1 = obj.call1(py, (op, py_int_node)); match r1 { Err(pyerr) => { - let eval_err: PyResult> = - eval_err_for_pyerr(py, &pyerr, arena.clone(), allocator); + let eval_err: PyResult> = eval_err_for_pyerr( + py, + &pyerr, + self.cache.clone(), + arena.clone(), + allocator, + ); let r: EvalErr = unwrap_or_eval_err(eval_err, args, "unexpected exception")?; Err(r) @@ -75,30 +84,21 @@ impl PyOperatorHandler { let py_node: &PyCell = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let node: i32 = PyNaNode::ptr(py_node, Some(py), arena.clone(), allocator); + let r = PyNaNode::ptr(py_node, py, &self.cache, &self.arena, allocator); + let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } } }) } - fn uncache(&self, py: Python, allocator: &mut IntAllocator, args: &i32) -> PyObject { + fn uncache<'p>(&'p self, py: Python<'p>, args: &i32) -> PyResult { let args = args.clone(); - let mut cache = self.cache.borrow_mut(); - match cache.get(&args) { - Some(obj) => obj.clone(), - None => { - let py_int_node: &PyCell = - PyCell::new(py, PyNaNode::new(self.arena.clone(), Some(args), None)).unwrap(); - // this hack ensures we have python representations in all children - PyNaNode::ensure_python_view(vec![py_int_node.to_object(py)], allocator, py); - let obj: PyObject = py_int_node.to_object(py); - cache.insert(args, obj.clone()); - obj - } - } + Ok(match from_cache(py, &self.cache, args)? { + Some(obj) => obj, + None => PyNaNode::from_ptr(py, &self.arena, args.clone())?.to_object(py), + }) } - */ } impl OperatorHandler for PyOperatorHandler { @@ -116,10 +116,6 @@ impl OperatorHandler for PyOperatorHandler { } } - // HACK TODO remove me - Err(EvalErr(0, "unknown op".to_string())) - - /* self.invoke_py_obj( self.py_callable.clone(), self.arena.clone(), @@ -128,7 +124,6 @@ impl OperatorHandler for PyOperatorHandler { args, max_cost, ) - */ } } diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 3d94e6a4..df7b51dc 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -21,7 +21,7 @@ pub fn new_cache(py: Python) -> PyResult { .to_object(py)) } -fn add_to_cache( +pub fn add_to_cache( py: Python, cache: &PyObject, ptr: ::Ptr, @@ -38,7 +38,7 @@ fn add_to_cache( Ok(py.run("cache[key] = value", None, Some(locals))?) } -fn from_cache( +pub fn from_cache( py: Python, cache: &PyObject, ptr: ::Ptr, From a25f7ff090a2b1f7f5e973dd408747845bededf8 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Sat, 13 Mar 2021 19:01:13 -0800 Subject: [PATCH 23/76] Call back into python seems to work now. --- src/py/op_fn.rs | 16 ++++++++++++++-- src/py/py_na_node.rs | 31 +++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index da69ea6b..89252917 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -1,11 +1,13 @@ use std::cell::RefCell; use std::collections::HashMap; +use pyo3::types::PyBytes; use pyo3::types::PyString; use pyo3::types::PyTuple; use pyo3::PyCell; use pyo3::PyErr; use pyo3::PyObject; +use pyo3::PyRefMut; use pyo3::PyResult; use pyo3::Python; use pyo3::ToPyObject; @@ -54,13 +56,23 @@ impl PyOperatorHandler { max_cost: Cost, ) -> Response<::Ptr> { Python::with_gil(|py| { - let op: &[u8] = allocator.buf(&op_buf); + let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); let r = self.uncache(py, args); let py_int_node = unwrap_or_eval_err(r, args, "can't uncache")?; + let mut py_na_node: PyRefMut = + unwrap_or_eval_err(py_int_node.extract(py), args, "can't convert")?; + if py_na_node.py_view.is_none() { + py_na_node.py_view = Some(unwrap_or_eval_err( + PyNaNode::py_view_for_allocator_ptr(py, &arena, allocator, args), + args, + "can't generate pyview", + )?); + } + drop(py_na_node); // TODO: implement a `populate_python_view` that accepts the borrowed `allocator` above // since the existing one will try to re-borrow it and fail // py_int_node.populate_python_view(py); - let r1 = obj.call1(py, (op, py_int_node)); + let r1 = obj.call1(py, (op, py_int_node.clone())); match r1 { Err(pyerr) => { diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index df7b51dc..018b9462 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -10,8 +10,8 @@ use super::py_view::PyView; #[pyclass(weakref, subclass)] pub struct PyNaNode { - py_view: Option, - native_view: Option, + pub py_view: Option, + pub native_view: Option, //int_arena_cache: PyObject, // WeakKeyDict[PyIntAllocator, int] } @@ -207,7 +207,16 @@ impl PyNaNode { //let mut py_int_allocator: PyRefMut = // native_view.arena.extract(py)?; //let mut allocator_to_use: &mut IntAllocator = &mut py_int_allocator.arena; - self.py_view = Some(Self::py_view_for_native_view(py, native_view)?); + let mut py_int_allocator: PyRefMut = + native_view.arena.extract(py)?; + let allocator: &mut IntAllocator = &mut py_int_allocator.arena; + + self.py_view = Some(Self::py_view_for_allocator_ptr( + py, + &native_view.arena, + allocator, + &native_view.ptr, + )?); } else { panic!("missing native AND python view"); } @@ -219,20 +228,22 @@ impl PyNaNode { py_raise(py, "no pyview available")? } - fn py_view_for_native_view(py: Python, native_view: &NativeView) -> PyResult { - let mut py_int_allocator: PyRefMut = native_view.arena.extract(py)?; - let allocator: &mut IntAllocator = &mut py_int_allocator.arena; - + pub fn py_view_for_allocator_ptr( + py: Python, + arena: &PyObject, + allocator: &mut IntAllocator, + ptr: &::Ptr, + ) -> PyResult { // create a PyView and return it - let py_view = match allocator.sexp(&native_view.ptr) { + let py_view = match allocator.sexp(ptr) { SExp::Atom(a) => { let blob = allocator.buf(&a); let py_bytes = PyBytes::new(py, blob); PyView::new_atom(py, py_bytes) } SExp::Pair(ptr_1, ptr_2) => { - let p1 = Self::from_ptr(py, &native_view.arena, ptr_1)?; - let p2 = Self::from_ptr(py, &native_view.arena, ptr_2)?; + let p1 = Self::from_ptr(py, arena, ptr_1)?; + let p2 = Self::from_ptr(py, arena, ptr_2)?; PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))? } }; From 45c30acd2e58a579c971e28ea5962b07c1aa3c43 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 15 Mar 2021 12:27:54 -0700 Subject: [PATCH 24/76] Fix problem with errors. --- src/py/api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/api.rs b/src/py/api.rs index 1455fc22..570ce58c 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -218,7 +218,7 @@ pub fn py_run_program( Ok((reduction.0, r.to_object(py))) } Err(eval_err) => { - let node: PyObject = eval_err.0.to_object(py); + let node: PyObject = PyNaNode::from_ptr(py, &arena_as_obj, eval_err.0)?.to_object(py); let s: String = eval_err.1; let s1: &str = &s; let msg: &PyString = PyString::new(py, s1); From fdcd422b299beec249701e4f8f0afb53b39eb637 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 15 Mar 2021 14:54:21 -0700 Subject: [PATCH 25/76] Checkpoint. --- src/py/op_fn.rs | 39 ++++++++++++++++++++------------------- src/py/py_na_node.rs | 2 +- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 89252917..bd09c677 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -57,23 +57,9 @@ impl PyOperatorHandler { ) -> Response<::Ptr> { Python::with_gil(|py| { let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); - let r = self.uncache(py, args); + let r = self.uncache(py, allocator, args); let py_int_node = unwrap_or_eval_err(r, args, "can't uncache")?; - let mut py_na_node: PyRefMut = - unwrap_or_eval_err(py_int_node.extract(py), args, "can't convert")?; - if py_na_node.py_view.is_none() { - py_na_node.py_view = Some(unwrap_or_eval_err( - PyNaNode::py_view_for_allocator_ptr(py, &arena, allocator, args), - args, - "can't generate pyview", - )?); - } - drop(py_na_node); - // TODO: implement a `populate_python_view` that accepts the borrowed `allocator` above - // since the existing one will try to re-borrow it and fail - // py_int_node.populate_python_view(py); let r1 = obj.call1(py, (op, py_int_node.clone())); - match r1 { Err(pyerr) => { let eval_err: PyResult> = eval_err_for_pyerr( @@ -104,12 +90,27 @@ impl PyOperatorHandler { }) } - fn uncache<'p>(&'p self, py: Python<'p>, args: &i32) -> PyResult { - let args = args.clone(); - Ok(match from_cache(py, &self.cache, args)? { + fn uncache<'p>( + &'p self, + py: Python<'p>, + allocator: &mut IntAllocator, + args: &i32, + ) -> PyResult { + let obj = match from_cache(py, &self.cache, args)? { Some(obj) => obj, None => PyNaNode::from_ptr(py, &self.arena, args.clone())?.to_object(py), - }) + }; + let mut py_na_node: PyRefMut = obj.extract(py)?; + if py_na_node.py_view.is_none() { + py_na_node.py_view = Some(PyNaNode::py_view_for_allocator_ptr( + py, &self.arena, allocator, args, + )?); + } + drop(py_na_node); + // TODO: implement a `populate_python_view` that accepts the borrowed `allocator` above + // since the existing one will try to re-borrow it and fail + // py_int_node.populate_python_view(py); + Ok(obj) } } diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 018b9462..b282bbce 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -41,7 +41,7 @@ pub fn add_to_cache( pub fn from_cache( py: Python, cache: &PyObject, - ptr: ::Ptr, + ptr: &::Ptr, ) -> PyResult> { let locals = [("cache", cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); py.eval("cache.get(key)", None, Some(locals))?.extract() From cff49ec16d337dd3571bebfe8a50d72434cb5e88 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 15 Mar 2021 16:45:11 -0700 Subject: [PATCH 26/76] Checkpoint. Call into python works a bit more. --- src/py/op_fn.rs | 21 +++---- src/py/py_na_node.rs | 137 ++++++++++++++++++++++++++++++------------- 2 files changed, 104 insertions(+), 54 deletions(-) diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index bd09c677..a185c2eb 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -96,20 +96,13 @@ impl PyOperatorHandler { allocator: &mut IntAllocator, args: &i32, ) -> PyResult { - let obj = match from_cache(py, &self.cache, args)? { - Some(obj) => obj, - None => PyNaNode::from_ptr(py, &self.arena, args.clone())?.to_object(py), - }; - let mut py_na_node: PyRefMut = obj.extract(py)?; - if py_na_node.py_view.is_none() { - py_na_node.py_view = Some(PyNaNode::py_view_for_allocator_ptr( - py, &self.arena, allocator, args, - )?); - } - drop(py_na_node); - // TODO: implement a `populate_python_view` that accepts the borrowed `allocator` above - // since the existing one will try to re-borrow it and fail - // py_int_node.populate_python_view(py); + let obj = PyNaNode::populate_python_view_with_allocator( + py, + &self.cache, + &self.arena, + allocator, + args, + )?; Ok(obj) } } diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index b282bbce..82d3e18d 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -47,6 +47,27 @@ pub fn from_cache( py.eval("cache.get(key)", None, Some(locals))?.extract() } +pub fn apply_to_tree(mut node: T, mut apply: F) -> PyResult<()> +where + F: FnMut(T) -> PyResult>, + T: Clone, +{ + let mut items = vec![node]; + loop { + let t = items.pop(); + if let Some(obj) = t { + if let Some((p0, p1)) = apply(obj.clone())? { + items.push(obj); + items.push(p0); + items.push(p1); + } + } else { + break; + } + } + Ok(()) +} + impl PyNaNode { fn new( py: Python, @@ -63,25 +84,23 @@ impl PyNaNode { } pub fn clear_native_view(slf: &PyCell, py: Python) -> PyResult<()> { - let mut items = vec![slf.to_object(py)]; - loop { - let t = items.pop(); - if let Some(obj) = t { - let mut node: PyRefMut = obj.extract(py)?; - node.populate_python_view(py)?; - assert!(node.py_view.is_some()); - node.native_view = None; - if let Some(PyView::Pair(tuple)) = &node.py_view { - let (p0, p1): (PyObject, PyObject) = tuple.extract(py)?; - //let (p0, p1): (&PyCell, &PyCell) = tuple.extract(py)?; - items.push(p0); - items.push(p1); + apply_to_tree(slf.to_object(py), move |obj: PyObject| { + let mut node: PyRefMut = obj.extract(py)?; + node.populate_python_view(py)?; + assert!(node.py_view.is_some()); + Ok(if let Some(PyView::Pair(tuple)) = &node.py_view { + let (p0, p1): (PyObject, PyObject) = tuple.extract(py)?; + if node.native_view.is_some() { + node.native_view = None; + Some((p0, p1)) + } else { + None } } else { - break; - } - } - Ok(()) + node.native_view = None; + None + }) + }) } pub fn add_to_cache(slf: &PyCell, py: Python, cache: &PyObject) -> PyResult<()> { @@ -124,24 +143,10 @@ impl PyNaNode { arena: &PyObject, allocator: &mut IntAllocator, ) -> PyResult<()> { - let mut to_cast: Vec = vec![slf.to_object(py)]; - loop { - let t: Option = to_cast.pop(); - match t { - None => break, - Some(node_ref) => { - let t1: &PyCell = node_ref.extract(py)?; - let transfer: Option<(PyObject, PyObject)> = - Self::add_to_native_cache(t1, py, arena, cache, allocator)?; - if let Some((p0, p1)) = transfer { - to_cast.push(node_ref); - to_cast.push(p0.to_object(py)); - to_cast.push(p1.to_object(py)); - } - } - } - } - Ok(()) + apply_to_tree(slf.to_object(py), move |node_ref: PyObject| { + let t1: &PyCell = node_ref.extract(py)?; + Self::add_to_native_cache(t1, py, arena, cache, allocator) + }) } /// This instance has a corresponding rep in some `IntAllocator` @@ -198,15 +203,11 @@ impl PyNaNode { } } - /// If this instance is using `NativeView`, replace it with an equivalent `PythonView` + /// If this instance has no `PythonView`, calculate it using the `NativeView` /// so it can be use from python. pub fn populate_python_view<'p>(&mut self, py: Python<'p>) -> PyResult<&PyView> { - // if using `NativeView`, swap it out for `PythonView` if self.py_view.is_none() { if let Some(native_view) = &self.native_view { - //let mut py_int_allocator: PyRefMut = - // native_view.arena.extract(py)?; - //let mut allocator_to_use: &mut IntAllocator = &mut py_int_allocator.arena; let mut py_int_allocator: PyRefMut = native_view.arena.extract(py)?; let allocator: &mut IntAllocator = &mut py_int_allocator.arena; @@ -228,6 +229,62 @@ impl PyNaNode { py_raise(py, "no pyview available")? } + /// If this instance has no `PythonView`, calculate it using the `NativeView` + /// so it can be use from python. + pub fn populate_python_view_with_allocator<'p>( + py: Python<'p>, + cache: &PyObject, + arena: &PyObject, + allocator: &mut IntAllocator, + ptr: &i32, + ) -> PyResult { + apply_to_tree(ptr.clone(), move |ptr| { + if from_cache(py, cache, &ptr)?.is_none() { + match allocator.sexp(&ptr) { + SExp::Atom(a) => { + let blob = allocator.buf(&a); + let py_bytes = PyBytes::new(py, blob); + + add_to_cache( + py, + cache, + ptr, + PyNaNode::new(py, Some(PyView::new_atom(py, py_bytes)), None)?, + )?; + Ok(None) + } + SExp::Pair(ptr_1, ptr_2) => { + let c1 = from_cache(py, cache, &ptr_1)?; + let c2 = from_cache(py, cache, &ptr_2)?; + if let (Some(p1), Some(p2)) = (c1, c2) { + add_to_cache( + py, + cache, + ptr.clone(), + PyNaNode::new( + py, + Some(PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?), + None, + )?, + )?; + Ok(None) + } else { + Ok(Some((ptr_1, ptr_2))) + } + } + } + } else { + Ok(None) + } + })?; + + if let Some(t) = from_cache(py, cache, &ptr)? { + Ok(t) + } else { + py_raise(py, "can't populate with py_view") + } + } + pub fn py_view_for_allocator_ptr( py: Python, arena: &PyObject, From 6d84a09ec9f88533e10e3cd3e105a035b0800559 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 15 Mar 2021 19:15:25 -0700 Subject: [PATCH 27/76] First crack at `py_native_mapping`. --- src/py/api.rs | 1 + src/py/mod.rs | 1 + src/py/py_na_node.rs | 2 +- src/py/py_native_mapping.rs | 250 ++++++++++++++++++++++++++++++++++++ 4 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 src/py/py_native_mapping.rs diff --git a/src/py/api.rs b/src/py/api.rs index 570ce58c..eeec98b6 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -199,6 +199,7 @@ pub fn py_run_program( let arena_as_obj = arena.to_object(py); PyNaNode::clear_native_view(program, py)?; let program = PyNaNode::ptr(program, py, &cache, &arena_as_obj, allocator)?; + PyNaNode::clear_native_view(args, py)?; let args = PyNaNode::ptr(args, py, &cache, &arena_as_obj, allocator)?; let op_lookup = Box::new(PyOperatorHandler::new( diff --git a/src/py/mod.rs b/src/py/mod.rs index 6da9ae88..f0531c07 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -8,6 +8,7 @@ pub mod native_view; pub mod op_fn; pub mod py_int_allocator; pub mod py_na_node; +pub mod py_native_mapping; pub mod py_node; pub mod py_view; pub mod run_program; diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 82d3e18d..e49274c3 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -69,7 +69,7 @@ where } impl PyNaNode { - fn new( + pub fn new( py: Python, py_view: Option, native_view: Option, diff --git a/src/py/py_native_mapping.rs b/src/py/py_native_mapping.rs new file mode 100644 index 00000000..c8170cd5 --- /dev/null +++ b/src/py/py_native_mapping.rs @@ -0,0 +1,250 @@ +use pyo3::prelude::*; +//use pyo3::pyclass::PyClassAlloc;` +use pyo3::types::{IntoPyDict, PyBytes, PyTuple, PyType}; + +use crate::allocator::{Allocator, SExp}; +use crate::int_allocator::IntAllocator; + +use super::native_view::NativeView; +use super::py_int_allocator::PyIntAllocator; +use super::py_view::PyView; + +use super::py_na_node::PyNaNode; + +pub struct PyNativeMapping { + py_to_native: PyObject, + native_to_py: PyObject, +} + +impl PyNativeMapping { + fn new(py: Python) -> PyResult { + let py_to_native = py + .eval("__import__('weakref').WeakKeyDictionary()", None, None)? + .to_object(py); + let native_to_py = py + .eval("__import__('weakref').WeakValueDictionary()", None, None)? + .to_object(py); + Ok(Self { + py_to_native, + native_to_py, + }) + } + + fn add( + &self, + py: Python, + obj: &PyCell, + ptr: &::Ptr, + ) -> PyResult<()> { + let locals = [ + ("py_to_native", self.py_to_native.clone()), + ("native_to_py", self.native_to_py.clone()), + ("obj", obj.to_object(py)), + ("ptr", ptr.to_object(py)), + ] + .into_py_dict(py); + + Ok(py.run( + "py_to_native[obj] = ptr; native_to_py[ptr] = obj", + None, + Some(locals), + )?) + } + + // py to native methods + + fn from_py_to_native_cache<'p>( + &'p self, + py: Python<'p>, + obj: &PyCell, + ) -> PyResult<::Ptr> { + let locals = [ + ("cache", self.py_to_native.clone()), + ("key", obj.to_object(py)), + ] + .into_py_dict(py); + py.eval("cache[key]", None, Some(locals))?.extract() + } + + fn populate_native( + &self, + py: Python, + obj: &PyCell, + allocator: &mut IntAllocator, + ) -> PyResult<::Ptr> { + apply_to_tree(obj.to_object(py), move |obj| { + let node: &PyCell = obj.extract(py)?; + + // is it in cache yet? + if self.from_py_to_native_cache(py, node).is_ok() { + // yep, we're done + return Ok(None); + } + + // it's not in the cache + + match &node.borrow().py_view { + Some(PyView::Atom(obj)) => { + let blob: &[u8] = obj.extract(py).unwrap(); + let ptr = allocator.new_atom(blob).unwrap(); + self.add(py, node, &ptr)?; + Ok(None) + } + Some(PyView::Pair(pair)) => { + let pair: &PyAny = pair.clone().into_ref(py); + let pair: &PyTuple = pair.extract()?; + let p0: &PyCell = pair.get_item(0).extract()?; + let p1: &PyCell = pair.get_item(1).extract()?; + let ptr_0 = match &p0.borrow().native_view { + Some(native_view) => Some(native_view.ptr), + None => None, + }; + let ptr_1 = match &p1.borrow().native_view { + Some(native_view) => Some(native_view.ptr), + None => None, + }; + if let (Some(ptr_0), Some(ptr_1)) = (ptr_0, ptr_1) { + let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); + self.add(py, node, &ptr)?; + Ok(None) + } else { + Ok(Some((p0.to_object(py), p1.to_object(py)))) + } + } + _ => py_raise(py, "py view is None"), + } + })?; + + self.from_py_to_native_cache(py, obj) + } + + pub fn native_for_py( + &self, + py: Python, + obj: &PyCell, + allocator: &mut IntAllocator, + ) -> PyResult<::Ptr> { + self.from_py_to_native_cache(py, obj) + .or_else(|err| self.populate_native(py, obj, allocator)) + } + + // native to py methods + + fn from_native_to_py_cache<'p>( + &'p self, + py: Python<'p>, + ptr: &::Ptr, + ) -> PyResult<&'p PyCell> { + let locals = [ + ("cache", self.native_to_py.clone()), + ("key", ptr.to_object(py)), + ] + .into_py_dict(py); + py.eval("cache[key]", None, Some(locals))?.extract() + } + + fn populate_python<'p>( + &'p self, + py: Python<'p>, + ptr: &::Ptr, + allocator: &mut IntAllocator, + ) -> PyResult<&'p PyCell> { + apply_to_tree(ptr.clone(), move |ptr| { + // is it in cache yet? + if self.from_native_to_py_cache(py, &ptr).is_ok() { + // yep, we're done + return Ok(None); + } + + // it's not in the cache + + match allocator.sexp(&ptr) { + SExp::Atom(a) => { + // it's an atom, so we just populate cache directly + let blob = allocator.buf(&a); + let py_bytes = PyBytes::new(py, blob); + self.add( + py, + PyNaNode::new(py, Some(PyView::new_atom(py, py_bytes)), None)?, + &ptr, + )?; + Ok(None) + } + SExp::Pair(ptr_1, ptr_2) => { + // we can only create this if the children are in the cache + // Let's fine out + let locals = [ + ("cache", self.native_to_py.clone()), + ("p1", ptr_1.to_object(py)), + ("p2", ptr_2.to_object(py)), + ] + .into_py_dict(py); + + let pair: PyResult<&PyAny> = + py.eval("(cache[p1], cache[p2])", None, Some(locals)); + + match pair { + // the children aren't in the cache, keep drilling down + Err(_) => Ok(Some((ptr_1, ptr_2))), + + // the children are in the cache, create new node & populate cache with it + Ok(tuple) => { + let (p1, p2): (&PyCell, &PyCell) = + tuple.extract()?; + self.add( + py, + PyNaNode::new( + py, + Some(PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?), + None, + )?, + &ptr, + )?; + Ok(None) + } + } + } + } + })?; + + self.from_native_to_py_cache(py, &ptr) + } + + pub fn py_for_native<'p>( + &'p self, + py: Python<'p>, + ptr: &::Ptr, + allocator: &mut IntAllocator, + ) -> PyResult<&'p PyCell> { + self.from_native_to_py_cache(py, ptr) + .or_else(|err| Ok(self.populate_python(py, ptr, allocator)?)) + } +} + +pub fn apply_to_tree(mut node: T, mut apply: F) -> PyResult<()> +where + F: FnMut(T) -> PyResult>, + T: Clone, +{ + let mut items = vec![node]; + loop { + let t = items.pop(); + if let Some(obj) = t { + if let Some((p0, p1)) = apply(obj.clone())? { + items.push(obj); + items.push(p0); + items.push(p1); + } + } else { + break; + } + } + Ok(()) +} + +fn py_raise(py: Python, msg: &str) -> PyResult { + let locals = [("msg", msg.to_object(py))].into_py_dict(py); + + py.run("raise RuntimeError(msg)", None, Some(locals))?; + panic!("we should never get here") +} From 73f42d7286c49526b00ba9c5847b3fb0b8168f34 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 15 Mar 2021 22:54:50 -0700 Subject: [PATCH 28/76] Lots of tests passing! --- src/py/api.rs | 4 +-- src/py/op_fn.rs | 56 +++++++++++++------------------------ src/py/py_na_node.rs | 7 +++++ src/py/py_native_mapping.rs | 52 +++++++++++++++------------------- 4 files changed, 52 insertions(+), 67 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index eeec98b6..1cb52268 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -203,11 +203,11 @@ pub fn py_run_program( let args = PyNaNode::ptr(args, py, &cache, &arena_as_obj, allocator)?; let op_lookup = Box::new(PyOperatorHandler::new( + py, opcode_lookup_by_name, arena.to_object(py), - cache, py_callback, - )); + )?); let r: Result, EvalErr> = crate::run_program::run_program( allocator, &program, &args, quote_kw, apply_kw, max_cost, op_lookup, None, diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index a185c2eb..8cc4c09e 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -16,32 +16,34 @@ use crate::allocator::Allocator; use crate::cost::Cost; use crate::int_allocator::IntAllocator; use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; -use crate::py::py_na_node::{add_to_cache, from_cache, PyNaNode}; +use crate::py::py_na_node::PyNaNode; use crate::reduction::{EvalErr, Reduction, Response}; use crate::run_program::OperatorHandler; +use super::py_native_mapping::PyNativeMapping; + pub struct PyOperatorHandler { native_lookup: FLookup, arena: PyObject, py_callable: PyObject, - cache: PyObject, + cache: PyNativeMapping, } impl PyOperatorHandler { pub fn new( + py: Python, opcode_lookup_by_name: HashMap>, arena: PyObject, - cache: PyObject, py_callable: PyObject, - ) -> Self { + ) -> PyResult { let native_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); - - PyOperatorHandler { + let cache = PyNativeMapping::new(py)?; + Ok(PyOperatorHandler { native_lookup, arena, py_callable, cache, - } + }) } } @@ -57,18 +59,16 @@ impl PyOperatorHandler { ) -> Response<::Ptr> { Python::with_gil(|py| { let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); - let r = self.uncache(py, allocator, args); - let py_int_node = unwrap_or_eval_err(r, args, "can't uncache")?; - let r1 = obj.call1(py, (op, py_int_node.clone())); + let r = unwrap_or_eval_err( + self.cache.py_for_native(py, args, allocator), + args, + "can't uncache", + )?; + let r1 = obj.call1(py, (op, r.to_object(py))); match r1 { Err(pyerr) => { - let eval_err: PyResult> = eval_err_for_pyerr( - py, - &pyerr, - self.cache.clone(), - arena.clone(), - allocator, - ); + let eval_err: PyResult> = + eval_err_for_pyerr(py, &pyerr, &self.cache, arena.clone(), allocator); let r: EvalErr = unwrap_or_eval_err(eval_err, args, "unexpected exception")?; Err(r) @@ -82,29 +82,13 @@ impl PyOperatorHandler { let py_node: &PyCell = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = PyNaNode::ptr(py_node, py, &self.cache, &self.arena, allocator); + let r = self.cache.native_for_py(py, py_node, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } } }) } - - fn uncache<'p>( - &'p self, - py: Python<'p>, - allocator: &mut IntAllocator, - args: &i32, - ) -> PyResult { - let obj = PyNaNode::populate_python_view_with_allocator( - py, - &self.cache, - &self.arena, - allocator, - args, - )?; - Ok(obj) - } } impl OperatorHandler for PyOperatorHandler { @@ -138,14 +122,14 @@ impl OperatorHandler for PyOperatorHandler { fn eval_err_for_pyerr<'p>( py: Python<'p>, pyerr: &PyErr, - cache: PyObject, + cache: &PyNativeMapping, arena: PyObject, allocator: &mut IntAllocator, ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = PyNaNode::ptr(&sexp, py, &cache, &arena, allocator)?; + let node: i32 = cache.native_for_py(py, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) } diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index e49274c3..4f0fe4a2 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -1,3 +1,5 @@ +use std::cell::Cell; + use pyo3::prelude::*; use pyo3::types::{IntoPyDict, PyBytes, PyTuple, PyType}; @@ -12,6 +14,7 @@ use super::py_view::PyView; pub struct PyNaNode { pub py_view: Option, pub native_view: Option, + pub int_cache: Cell>, //int_arena_cache: PyObject, // WeakKeyDict[PyIntAllocator, int] } @@ -74,11 +77,13 @@ impl PyNaNode { py_view: Option, native_view: Option, ) -> PyResult<&PyCell> { + let int_cache = Cell::new(None); PyCell::new( py, PyNaNode { py_view, native_view, + int_cache, }, ) } @@ -317,6 +322,7 @@ impl PyNaNode { Self { py_view: Some(py_view), native_view: None, + int_cache: Cell::new(None), } } else { let py_bytes: &PyBytes = obj.extract()?; @@ -324,6 +330,7 @@ impl PyNaNode { Self { py_view: Some(py_view), native_view: None, + int_cache: Cell::new(None), } }) } diff --git a/src/py/py_native_mapping.rs b/src/py/py_native_mapping.rs index c8170cd5..652631ab 100644 --- a/src/py/py_native_mapping.rs +++ b/src/py/py_native_mapping.rs @@ -12,22 +12,16 @@ use super::py_view::PyView; use super::py_na_node::PyNaNode; pub struct PyNativeMapping { - py_to_native: PyObject, native_to_py: PyObject, } impl PyNativeMapping { - fn new(py: Python) -> PyResult { - let py_to_native = py - .eval("__import__('weakref').WeakKeyDictionary()", None, None)? - .to_object(py); + pub fn new(py: Python) -> PyResult { let native_to_py = py - .eval("__import__('weakref').WeakValueDictionary()", None, None)? + //.eval("__import__('weakref').WeakValueDictionary()", None, None)? + .eval("dict()", None, None)? .to_object(py); - Ok(Self { - py_to_native, - native_to_py, - }) + Ok(Self { native_to_py }) } fn add( @@ -36,19 +30,17 @@ impl PyNativeMapping { obj: &PyCell, ptr: &::Ptr, ) -> PyResult<()> { + obj.borrow().int_cache.set(Some(ptr.clone())); + let locals = [ - ("py_to_native", self.py_to_native.clone()), ("native_to_py", self.native_to_py.clone()), ("obj", obj.to_object(py)), ("ptr", ptr.to_object(py)), ] .into_py_dict(py); - Ok(py.run( - "py_to_native[obj] = ptr; native_to_py[ptr] = obj", - None, - Some(locals), - )?) + let r = py.run("native_to_py[ptr] = obj", None, Some(locals)); + r } // py to native methods @@ -58,12 +50,18 @@ impl PyNativeMapping { py: Python<'p>, obj: &PyCell, ) -> PyResult<::Ptr> { + let ptr: Option = obj.borrow().int_cache.get(); let locals = [ - ("cache", self.py_to_native.clone()), - ("key", obj.to_object(py)), + ("cache", self.native_to_py.clone()), + ("key", ptr.to_object(py)), ] .into_py_dict(py); - py.eval("cache[key]", None, Some(locals))?.extract() + let obj1: &PyCell = py.eval("cache[key]", None, Some(locals))?.extract()?; + if obj1.to_object(py) == obj.to_object(py) { + Ok(ptr.unwrap()) + } else { + py_raise(py, "not in native cache") + } } fn populate_native( @@ -88,6 +86,7 @@ impl PyNativeMapping { let blob: &[u8] = obj.extract(py).unwrap(); let ptr = allocator.new_atom(blob).unwrap(); self.add(py, node, &ptr)?; + Ok(None) } Some(PyView::Pair(pair)) => { @@ -95,16 +94,10 @@ impl PyNativeMapping { let pair: &PyTuple = pair.extract()?; let p0: &PyCell = pair.get_item(0).extract()?; let p1: &PyCell = pair.get_item(1).extract()?; - let ptr_0 = match &p0.borrow().native_view { - Some(native_view) => Some(native_view.ptr), - None => None, - }; - let ptr_1 = match &p1.borrow().native_view { - Some(native_view) => Some(native_view.ptr), - None => None, - }; + let ptr_0 = &p0.borrow().int_cache.get(); + let ptr_1 = &p1.borrow().int_cache.get(); if let (Some(ptr_0), Some(ptr_1)) = (ptr_0, ptr_1) { - let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); + let ptr = allocator.new_pair(*ptr_0, *ptr_1).unwrap(); self.add(py, node, &ptr)?; Ok(None) } else { @@ -115,7 +108,8 @@ impl PyNativeMapping { } })?; - self.from_py_to_native_cache(py, obj) + let r = self.from_py_to_native_cache(py, obj); + r } pub fn native_for_py( From 44d839ff3a908e22a58d3dea98c951ce69eaa401 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 16 Mar 2021 23:33:25 -0700 Subject: [PATCH 29/76] More tests passing! --- src/py/api.rs | 32 ++-- src/py/mod.rs | 2 +- src/py/native_view.rs | 2 + src/py/op_fn.rs | 27 +-- src/py/py_na_node.rs | 283 ++--------------------------- src/py/py_native_mapping.rs | 352 ++++++++++++++++++------------------ 6 files changed, 224 insertions(+), 474 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 1cb52268..8f1ea986 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -9,6 +9,7 @@ use pyo3::PyObject; use super::py_int_allocator::PyIntAllocator; use super::py_na_node::{new_cache, PyNaNode}; +use super::py_native_mapping::{add, native_for_py, new_mapping, py_for_native}; //use super::run_program::{__pyo3_get_function_deserialize_and_run_program, STRICT_MODE}; use crate::int_allocator::IntAllocator; @@ -120,8 +121,9 @@ const fn allocator_for_py(_py: Python) -> AllocatorT { fn serialize_from_bytes<'p>(py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyCell> { let py_int_allocator = PyCell::new(py, PyIntAllocator::default())?; let allocator: &mut IntAllocator = &mut py_int_allocator.borrow_mut().arena; + let cache = new_mapping(py)?; let ptr = node_from_bytes(allocator, blob)?; - PyNaNode::from_ptr(py, &py_int_allocator.to_object(py), ptr) + py_for_native(py, &cache, &ptr, allocator) } use crate::node::Node; @@ -134,13 +136,9 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<& let py_int_allocator: &mut PyIntAllocator = &mut py_int_allocator_cell.borrow_mut(); let allocator: &mut IntAllocator = &mut py_int_allocator.arena; - let ptr = PyNaNode::ptr( - sexp, - py, - &new_cache(py)?, - &py_int_allocator_cell.to_object(py), - allocator, - )?; + let mapping = new_mapping(py)?; + + let ptr = native_for_py(py, &mapping, sexp, allocator)?; let node = Node::new(allocator, ptr); let s: Vec = node_to_bytes(&node)?; @@ -186,7 +184,6 @@ pub fn py_run_program( opcode_lookup_by_name: HashMap>, py_callback: PyObject, ) -> PyResult<(Cost, PyObject)> { - let cache = new_cache(py)?; let arena = PyCell::new( py, PyIntAllocator { @@ -197,17 +194,19 @@ pub fn py_run_program( let allocator: &mut IntAllocator = &mut arena_ref.arena; let arena_as_obj = arena.to_object(py); - PyNaNode::clear_native_view(program, py)?; - let program = PyNaNode::ptr(program, py, &cache, &arena_as_obj, allocator)?; - PyNaNode::clear_native_view(args, py)?; - let args = PyNaNode::ptr(args, py, &cache, &arena_as_obj, allocator)?; - + let cache = new_mapping(py)?; let op_lookup = Box::new(PyOperatorHandler::new( py, opcode_lookup_by_name, arena.to_object(py), py_callback, + cache.clone(), )?); + PyNaNode::clear_native_view(program, py); + let program = native_for_py(py, &cache, program, allocator)?; + + PyNaNode::clear_native_view(args, py); + let args = native_for_py(py, &cache, args, allocator)?; let r: Result, EvalErr> = crate::run_program::run_program( allocator, &program, &args, quote_kw, apply_kw, max_cost, op_lookup, None, @@ -215,11 +214,12 @@ pub fn py_run_program( match r { Ok(reduction) => { - let r = PyNaNode::from_ptr(py, &arena.to_object(py), reduction.1)?; + let r = py_for_native(py, &cache, &reduction.1, allocator)?; + PyNaNode::clear_native_view(r, py); Ok((reduction.0, r.to_object(py))) } Err(eval_err) => { - let node: PyObject = PyNaNode::from_ptr(py, &arena_as_obj, eval_err.0)?.to_object(py); + let node: PyObject = py_for_native(py, &cache, &eval_err.0, allocator)?.to_object(py); let s: String = eval_err.1; let s1: &str = &s; let msg: &PyString = PyString::new(py, s1); diff --git a/src/py/mod.rs b/src/py/mod.rs index f0531c07..9ce091a1 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -4,7 +4,7 @@ pub mod f_table; pub mod glue; //pub mod int_allocator_gateway; pub mod native_op_lookup; -pub mod native_view; +//pub mod native_view; pub mod op_fn; pub mod py_int_allocator; pub mod py_na_node; diff --git a/src/py/native_view.rs b/src/py/native_view.rs index 3edab6d7..01ecb94f 100644 --- a/src/py/native_view.rs +++ b/src/py/native_view.rs @@ -1,3 +1,4 @@ +/* use pyo3::prelude::PyObject; use crate::allocator::Allocator; @@ -14,3 +15,4 @@ impl NativeView { NativeView { arena, ptr } } } +*/ \ No newline at end of file diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 8cc4c09e..152057b5 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -15,18 +15,18 @@ use pyo3::ToPyObject; use crate::allocator::Allocator; use crate::cost::Cost; use crate::int_allocator::IntAllocator; -use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; -use crate::py::py_na_node::PyNaNode; use crate::reduction::{EvalErr, Reduction, Response}; use crate::run_program::OperatorHandler; -use super::py_native_mapping::PyNativeMapping; +use super::f_table::{f_lookup_for_hashmap, FLookup}; +use super::py_na_node::PyNaNode; +use super::py_native_mapping::{native_for_py, py_for_native}; pub struct PyOperatorHandler { native_lookup: FLookup, arena: PyObject, py_callable: PyObject, - cache: PyNativeMapping, + pub cache: PyObject, } impl PyOperatorHandler { @@ -35,9 +35,9 @@ impl PyOperatorHandler { opcode_lookup_by_name: HashMap>, arena: PyObject, py_callable: PyObject, + cache: PyObject, ) -> PyResult { let native_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); - let cache = PyNativeMapping::new(py)?; Ok(PyOperatorHandler { native_lookup, arena, @@ -60,15 +60,20 @@ impl PyOperatorHandler { Python::with_gil(|py| { let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); let r = unwrap_or_eval_err( - self.cache.py_for_native(py, args, allocator), + py_for_native(py, &self.cache, args, allocator), args, "can't uncache", )?; let r1 = obj.call1(py, (op, r.to_object(py))); match r1 { Err(pyerr) => { - let eval_err: PyResult> = - eval_err_for_pyerr(py, &pyerr, &self.cache, arena.clone(), allocator); + let eval_err: PyResult> = eval_err_for_pyerr( + py, + &pyerr, + self.cache.clone(), + arena.clone(), + allocator, + ); let r: EvalErr = unwrap_or_eval_err(eval_err, args, "unexpected exception")?; Err(r) @@ -82,7 +87,7 @@ impl PyOperatorHandler { let py_node: &PyCell = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = self.cache.native_for_py(py, py_node, allocator); + let r = native_for_py(py, &self.cache, py_node, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } @@ -122,14 +127,14 @@ impl OperatorHandler for PyOperatorHandler { fn eval_err_for_pyerr<'p>( py: Python<'p>, pyerr: &PyErr, - cache: &PyNativeMapping, + cache: PyObject, arena: PyObject, allocator: &mut IntAllocator, ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = cache.native_for_py(py, sexp, allocator)?; + let node: i32 = native_for_py(py, &cache, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) } diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 4f0fe4a2..80507551 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -6,14 +6,12 @@ use pyo3::types::{IntoPyDict, PyBytes, PyTuple, PyType}; use crate::allocator::{Allocator, SExp}; use crate::int_allocator::IntAllocator; -use super::native_view::NativeView; use super::py_int_allocator::PyIntAllocator; use super::py_view::PyView; #[pyclass(weakref, subclass)] pub struct PyNaNode { pub py_view: Option, - pub native_view: Option, pub int_cache: Cell>, //int_arena_cache: PyObject, // WeakKeyDict[PyIntAllocator, int] } @@ -72,245 +70,29 @@ where } impl PyNaNode { - pub fn new( - py: Python, - py_view: Option, - native_view: Option, - ) -> PyResult<&PyCell> { + pub fn new(py: Python, py_view: Option) -> PyResult<&PyCell> { let int_cache = Cell::new(None); - PyCell::new( - py, - PyNaNode { - py_view, - native_view, - int_cache, - }, - ) + PyCell::new(py, PyNaNode { py_view, int_cache }) } pub fn clear_native_view(slf: &PyCell, py: Python) -> PyResult<()> { apply_to_tree(slf.to_object(py), move |obj: PyObject| { let mut node: PyRefMut = obj.extract(py)?; - node.populate_python_view(py)?; assert!(node.py_view.is_some()); Ok(if let Some(PyView::Pair(tuple)) = &node.py_view { let (p0, p1): (PyObject, PyObject) = tuple.extract(py)?; - if node.native_view.is_some() { - node.native_view = None; + if node.int_cache.get().is_some() { + node.int_cache.set(None); Some((p0, p1)) } else { None } } else { - node.native_view = None; + node.int_cache.set(None); None }) }) } - - pub fn add_to_cache(slf: &PyCell, py: Python, cache: &PyObject) -> PyResult<()> { - if let Some(native_view) = &slf.borrow().native_view { - add_to_cache(py, cache, native_view.ptr, slf)?; - } - Ok(()) - } - - pub fn from_ptr<'p>( - py: Python<'p>, - py_int_allocator: &PyObject, - ptr: ::Ptr, - ) -> PyResult<&'p PyCell> { - let native_view = NativeView::new(py_int_allocator.clone(), ptr); - Self::new(py, None, Some(native_view)) - } - - pub fn ptr( - slf: &PyCell, - py: Python, - cache: &PyObject, - arena: &PyObject, - allocator: &mut IntAllocator, - ) -> PyResult<::Ptr> { - // check if we need to clear the native view - // if arena doesn't match, clear native view - Self::populate_native_view(slf, py, cache, arena, allocator)?; - if let Some(native_view) = &slf.borrow().native_view { - Ok(native_view.ptr) - } else { - py_raise(py, "oops")? - } - } - - pub fn populate_native_view<'p>( - slf: &PyCell, - py: Python<'p>, - cache: &PyObject, - arena: &PyObject, - allocator: &mut IntAllocator, - ) -> PyResult<()> { - apply_to_tree(slf.to_object(py), move |node_ref: PyObject| { - let t1: &PyCell = node_ref.extract(py)?; - Self::add_to_native_cache(t1, py, arena, cache, allocator) - }) - } - - /// This instance has a corresponding rep in some `IntAllocator` - /// Notate this in the cache. - fn add_to_native_cache<'p>( - slf_cell: &PyCell, - py: Python<'p>, - arena: &PyObject, - cache: &PyObject, - allocator: &mut IntAllocator, - ) -> PyResult> { - // if it's an atom, we add it to the allocator & cache the addition - // if it's a pair, and BOTH are in the cache, we add to allocator & cache - // otherwise, we return both children so they can be cached (if necessary) - let mut slf = slf_cell.borrow_mut(); - let slf: &mut PyNaNode = &mut slf; - if slf.native_view.is_none() { - let py_view = slf.populate_python_view(py)?; - let new_ptr = { - match py_view { - PyView::Atom(obj) => { - let blob: &[u8] = obj.extract(py).unwrap(); - let ptr = allocator.new_atom(blob).unwrap(); - add_to_cache(py, cache, ptr, slf_cell)?; - ptr - } - PyView::Pair(pair) => { - let pair: &'p PyAny = pair.clone().into_ref(py); - let pair: &'p PyTuple = pair.extract()?; - let p0: &'p PyCell = pair.get_item(0).extract()?; - let p1: &'p PyCell = pair.get_item(1).extract()?; - let ptr_0 = match &p0.borrow().native_view { - Some(native_view) => Some(native_view.ptr), - None => None, - }; - let ptr_1 = match &p1.borrow().native_view { - Some(native_view) => Some(native_view.ptr), - None => None, - }; - if let (Some(ptr_0), Some(ptr_1)) = (ptr_0, ptr_1) { - let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); - add_to_cache(py, cache, ptr, slf_cell)?; - ptr - } else { - return Ok(Some((p0.to_object(py), p1.to_object(py)))); - } - } - } - }; - slf.native_view = Some(NativeView::new(arena.clone(), new_ptr)); - Ok(None) - } else { - Ok(None) - } - } - - /// If this instance has no `PythonView`, calculate it using the `NativeView` - /// so it can be use from python. - pub fn populate_python_view<'p>(&mut self, py: Python<'p>) -> PyResult<&PyView> { - if self.py_view.is_none() { - if let Some(native_view) = &self.native_view { - let mut py_int_allocator: PyRefMut = - native_view.arena.extract(py)?; - let allocator: &mut IntAllocator = &mut py_int_allocator.arena; - - self.py_view = Some(Self::py_view_for_allocator_ptr( - py, - &native_view.arena, - allocator, - &native_view.ptr, - )?); - } else { - panic!("missing native AND python view"); - } - } - match &self.py_view { - Some(py_view) => return Ok(&py_view), - None => (), - }; - py_raise(py, "no pyview available")? - } - - /// If this instance has no `PythonView`, calculate it using the `NativeView` - /// so it can be use from python. - pub fn populate_python_view_with_allocator<'p>( - py: Python<'p>, - cache: &PyObject, - arena: &PyObject, - allocator: &mut IntAllocator, - ptr: &i32, - ) -> PyResult { - apply_to_tree(ptr.clone(), move |ptr| { - if from_cache(py, cache, &ptr)?.is_none() { - match allocator.sexp(&ptr) { - SExp::Atom(a) => { - let blob = allocator.buf(&a); - let py_bytes = PyBytes::new(py, blob); - - add_to_cache( - py, - cache, - ptr, - PyNaNode::new(py, Some(PyView::new_atom(py, py_bytes)), None)?, - )?; - Ok(None) - } - SExp::Pair(ptr_1, ptr_2) => { - let c1 = from_cache(py, cache, &ptr_1)?; - let c2 = from_cache(py, cache, &ptr_2)?; - if let (Some(p1), Some(p2)) = (c1, c2) { - add_to_cache( - py, - cache, - ptr.clone(), - PyNaNode::new( - py, - Some(PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?), - None, - )?, - )?; - Ok(None) - } else { - Ok(Some((ptr_1, ptr_2))) - } - } - } - } else { - Ok(None) - } - })?; - - if let Some(t) = from_cache(py, cache, &ptr)? { - Ok(t) - } else { - py_raise(py, "can't populate with py_view") - } - } - - pub fn py_view_for_allocator_ptr( - py: Python, - arena: &PyObject, - allocator: &mut IntAllocator, - ptr: &::Ptr, - ) -> PyResult { - // create a PyView and return it - let py_view = match allocator.sexp(ptr) { - SExp::Atom(a) => { - let blob = allocator.buf(&a); - let py_bytes = PyBytes::new(py, blob); - PyView::new_atom(py, py_bytes) - } - SExp::Pair(ptr_1, ptr_2) => { - let p1 = Self::from_ptr(py, arena, ptr_1)?; - let p2 = Self::from_ptr(py, arena, ptr_2)?; - PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))? - } - }; - Ok(py_view) - } } #[pymethods] @@ -321,7 +103,6 @@ impl PyNaNode { let py_view = PyView::new_pair(py, tuple)?; Self { py_view: Some(py_view), - native_view: None, int_cache: Cell::new(None), } } else { @@ -329,7 +110,6 @@ impl PyNaNode { let py_view = PyView::new_atom(py, py_bytes); Self { py_view: Some(py_view), - native_view: None, int_cache: Cell::new(None), } }) @@ -338,7 +118,7 @@ impl PyNaNode { #[classmethod] fn new_atom<'p>(_cls: &PyType, py: Python<'p>, atom: &PyBytes) -> PyResult<&'p PyCell> { let py_view = PyView::new_atom(py, atom); - Self::new(py, Some(py_view), None) + Self::new(py, Some(py_view)) } #[classmethod] @@ -350,48 +130,34 @@ impl PyNaNode { ) -> PyResult<&'p PyCell> { let tuple = PyTuple::new(py, &[p1, p2]); let py_view = PyView::new_pair(py, tuple)?; - Self::new(py, Some(py_view), None) + Self::new(py, Some(py_view)) } #[classmethod] fn new_tuple<'p>(_cls: &PyType, py: Python<'p>, tuple: &PyTuple) -> PyResult<&'p PyCell> { let py_view = PyView::new_pair(py, tuple)?; - Self::new(py, Some(py_view), None) + Self::new(py, Some(py_view)) } #[getter(atom)] pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - let mut slf = slf.try_borrow_mut()?; - let py_view: &PyView = slf.populate_python_view(py)?; - match py_view { - PyView::Atom(obj) => Ok(obj.clone()), + match slf.try_borrow()?.py_view.as_ref() { + Some(PyView::Atom(obj)) => Ok(obj.clone()), _ => Ok(py.None()), } } #[getter(pair)] pub fn pair<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - let mut slf = slf.try_borrow_mut()?; - let py_view = slf.populate_python_view(py)?; - match py_view { - PyView::Pair(obj) => Ok(obj.clone()), + match slf.try_borrow()?.py_view.as_ref() { + Some(PyView::Pair(obj)) => Ok(obj.clone()), _ => Ok(py.None()), } } #[getter(native)] pub fn native<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - Ok(match &slf.borrow().native_view { - Some(native_view) => { - let locals = [ - ("a", native_view.arena.clone()), - ("b", native_view.ptr.into_py(py)), - ] - .into_py_dict(py); - py.eval("(a, b)", None, Some(locals))?.to_object(py) - } - None => py.None(), - }) + Ok(slf.borrow().int_cache.get().to_object(py)) } #[getter(python)] @@ -402,27 +168,4 @@ impl PyNaNode { _ => py.None(), }) } - - pub fn ensure_python(slf: &PyCell, py: Python) -> PyResult<()> { - Self::populate_python_view(&mut slf.borrow_mut(), py)?; - Ok(()) - } - - pub fn ensure_native( - slf: &PyCell, - py: Python, - cache: PyObject, - arena: PyObject, - ) -> PyResult<()> { - let py_int_allocator: &PyCell = arena.extract(py)?; - let allocator: &mut IntAllocator = &mut py_int_allocator.borrow_mut().arena; - Self::populate_native_view(slf, py, &cache, &arena, allocator) - } -} - -fn py_raise(py: Python, msg: &str) -> PyResult { - let locals = [("msg", msg.to_object(py))].into_py_dict(py); - - py.run("raise RuntimeError(msg)", None, Some(locals))?; - panic!("we should never get here") } diff --git a/src/py/py_native_mapping.rs b/src/py/py_native_mapping.rs index 652631ab..17eb3958 100644 --- a/src/py/py_native_mapping.rs +++ b/src/py/py_native_mapping.rs @@ -5,217 +5,217 @@ use pyo3::types::{IntoPyDict, PyBytes, PyTuple, PyType}; use crate::allocator::{Allocator, SExp}; use crate::int_allocator::IntAllocator; -use super::native_view::NativeView; use super::py_int_allocator::PyIntAllocator; use super::py_view::PyView; use super::py_na_node::PyNaNode; -pub struct PyNativeMapping { - native_to_py: PyObject, -} - -impl PyNativeMapping { - pub fn new(py: Python) -> PyResult { - let native_to_py = py - //.eval("__import__('weakref').WeakValueDictionary()", None, None)? - .eval("dict()", None, None)? - .to_object(py); - Ok(Self { native_to_py }) - } - +/* +pub trait PyNativeMapping { fn add( &self, py: Python, obj: &PyCell, ptr: &::Ptr, - ) -> PyResult<()> { - obj.borrow().int_cache.set(Some(ptr.clone())); - - let locals = [ - ("native_to_py", self.native_to_py.clone()), - ("obj", obj.to_object(py)), - ("ptr", ptr.to_object(py)), - ] - .into_py_dict(py); - - let r = py.run("native_to_py[ptr] = obj", None, Some(locals)); - r - } + ) -> PyResult<()>; - // py to native methods - - fn from_py_to_native_cache<'p>( - &'p self, - py: Python<'p>, - obj: &PyCell, - ) -> PyResult<::Ptr> { - let ptr: Option = obj.borrow().int_cache.get(); - let locals = [ - ("cache", self.native_to_py.clone()), - ("key", ptr.to_object(py)), - ] - .into_py_dict(py); - let obj1: &PyCell = py.eval("cache[key]", None, Some(locals))?.extract()?; - if obj1.to_object(py) == obj.to_object(py) { - Ok(ptr.unwrap()) - } else { - py_raise(py, "not in native cache") - } - } - - fn populate_native( + fn native_for_py( &self, py: Python, obj: &PyCell, allocator: &mut IntAllocator, - ) -> PyResult<::Ptr> { - apply_to_tree(obj.to_object(py), move |obj| { - let node: &PyCell = obj.extract(py)?; - - // is it in cache yet? - if self.from_py_to_native_cache(py, node).is_ok() { - // yep, we're done - return Ok(None); - } + ) -> PyResult<::Ptr>; - // it's not in the cache + fn py_for_native<'p>( + &'p self, + py: Python<'p>, + ptr: &::Ptr, + allocator: &mut IntAllocator, + ) -> PyResult<&'p PyCell>; +} +*/ + +pub fn new_mapping(py: Python) -> PyResult { + Ok(py + //.eval("__import__('weakref').WeakValueDictionary()", None, None)? + .eval("dict()", None, None)? + .to_object(py)) +} + +pub fn add( + py: Python, + cache: &PyObject, + obj: &PyCell, + ptr: &::Ptr, +) -> PyResult<()> { + //obj.borrow().int_cache.set(Some(ptr.clone())); + + let locals = [ + ("cache", cache.clone()), + ("obj", obj.to_object(py)), + ("ptr", ptr.to_object(py)), + ] + .into_py_dict(py); + + py.run("cache[ptr] = obj; cache[obj] = ptr", None, Some(locals)) +} + +// py to native methods + +fn from_py_to_native_cache<'p>( + py: Python<'p>, + cache: &PyObject, + obj: &PyCell, +) -> PyResult<::Ptr> { + let locals = [("cache", cache.clone()), ("key", obj.to_object(py))].into_py_dict(py); + py.eval("cache.get(key)", None, Some(locals))?.extract() +} + +fn populate_native( + py: Python, + cache: &PyObject, + obj: &PyCell, + allocator: &mut IntAllocator, +) -> PyResult<::Ptr> { + apply_to_tree(obj.to_object(py), move |obj| { + let node: &PyCell = obj.extract(py)?; + + // is it in cache yet? + if from_py_to_native_cache(py, cache, node).is_ok() { + // yep, we're done + return Ok(None); + } + + // it's not in the cache - match &node.borrow().py_view { - Some(PyView::Atom(obj)) => { - let blob: &[u8] = obj.extract(py).unwrap(); - let ptr = allocator.new_atom(blob).unwrap(); - self.add(py, node, &ptr)?; + match &node.borrow().py_view { + Some(PyView::Atom(obj)) => { + let blob: &[u8] = obj.extract(py).unwrap(); + let ptr = allocator.new_atom(blob).unwrap(); + add(py, cache, node, &ptr)?; + Ok(None) + } + Some(PyView::Pair(pair)) => { + let pair: &PyAny = pair.clone().into_ref(py); + let pair: &PyTuple = pair.extract()?; + let p0: &PyCell = pair.get_item(0).extract()?; + let p1: &PyCell = pair.get_item(1).extract()?; + let ptr_0: PyResult = from_py_to_native_cache(py, cache, p0); + let ptr_1: PyResult = from_py_to_native_cache(py, cache, p1); + if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { + let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); + add(py, cache, node, &ptr)?; Ok(None) + } else { + Ok(Some((p0.to_object(py), p1.to_object(py)))) } - Some(PyView::Pair(pair)) => { - let pair: &PyAny = pair.clone().into_ref(py); - let pair: &PyTuple = pair.extract()?; - let p0: &PyCell = pair.get_item(0).extract()?; - let p1: &PyCell = pair.get_item(1).extract()?; - let ptr_0 = &p0.borrow().int_cache.get(); - let ptr_1 = &p1.borrow().int_cache.get(); - if let (Some(ptr_0), Some(ptr_1)) = (ptr_0, ptr_1) { - let ptr = allocator.new_pair(*ptr_0, *ptr_1).unwrap(); - self.add(py, node, &ptr)?; - Ok(None) - } else { - Ok(Some((p0.to_object(py), p1.to_object(py)))) - } - } - _ => py_raise(py, "py view is None"), } - })?; + _ => py_raise(py, "py view is None"), + } + })?; - let r = self.from_py_to_native_cache(py, obj); - r - } + from_py_to_native_cache(py, cache, obj) +} - pub fn native_for_py( - &self, - py: Python, - obj: &PyCell, - allocator: &mut IntAllocator, - ) -> PyResult<::Ptr> { - self.from_py_to_native_cache(py, obj) - .or_else(|err| self.populate_native(py, obj, allocator)) - } +pub fn native_for_py( + py: Python, + cache: &PyObject, + obj: &PyCell, + allocator: &mut IntAllocator, +) -> PyResult<::Ptr> { + from_py_to_native_cache(py, cache, obj) + .or_else(|err| populate_native(py, cache, obj, allocator)) +} - // native to py methods +// native to py methods - fn from_native_to_py_cache<'p>( - &'p self, - py: Python<'p>, - ptr: &::Ptr, - ) -> PyResult<&'p PyCell> { - let locals = [ - ("cache", self.native_to_py.clone()), - ("key", ptr.to_object(py)), - ] - .into_py_dict(py); - py.eval("cache[key]", None, Some(locals))?.extract() - } +fn from_native_to_py_cache<'p>( + py: Python<'p>, + cache: &PyObject, + ptr: &::Ptr, +) -> PyResult<&'p PyCell> { + let locals = [("cache", cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); + py.eval("cache[key]", None, Some(locals))?.extract() +} - fn populate_python<'p>( - &'p self, - py: Python<'p>, - ptr: &::Ptr, - allocator: &mut IntAllocator, - ) -> PyResult<&'p PyCell> { - apply_to_tree(ptr.clone(), move |ptr| { - // is it in cache yet? - if self.from_native_to_py_cache(py, &ptr).is_ok() { - // yep, we're done - return Ok(None); - } +fn populate_python<'p>( + py: Python<'p>, + cache: &PyObject, + ptr: &::Ptr, + allocator: &mut IntAllocator, +) -> PyResult<&'p PyCell> { + apply_to_tree(ptr.clone(), move |ptr| { + // is it in cache yet? + if from_native_to_py_cache(py, cache, &ptr).is_ok() { + // yep, we're done + return Ok(None); + } - // it's not in the cache - - match allocator.sexp(&ptr) { - SExp::Atom(a) => { - // it's an atom, so we just populate cache directly - let blob = allocator.buf(&a); - let py_bytes = PyBytes::new(py, blob); - self.add( - py, - PyNaNode::new(py, Some(PyView::new_atom(py, py_bytes)), None)?, - &ptr, - )?; - Ok(None) - } - SExp::Pair(ptr_1, ptr_2) => { - // we can only create this if the children are in the cache - // Let's fine out - let locals = [ - ("cache", self.native_to_py.clone()), - ("p1", ptr_1.to_object(py)), - ("p2", ptr_2.to_object(py)), - ] - .into_py_dict(py); - - let pair: PyResult<&PyAny> = - py.eval("(cache[p1], cache[p2])", None, Some(locals)); - - match pair { - // the children aren't in the cache, keep drilling down - Err(_) => Ok(Some((ptr_1, ptr_2))), - - // the children are in the cache, create new node & populate cache with it - Ok(tuple) => { - let (p1, p2): (&PyCell, &PyCell) = - tuple.extract()?; - self.add( + // it's not in the cache + + match allocator.sexp(&ptr) { + SExp::Atom(a) => { + // it's an atom, so we just populate cache directly + let blob = allocator.buf(&a); + let py_bytes = PyBytes::new(py, blob); + add( + py, + cache, + PyNaNode::new(py, Some(PyView::new_atom(py, py_bytes)))?, + &ptr, + )?; + Ok(None) + } + SExp::Pair(ptr_1, ptr_2) => { + // we can only create this if the children are in the cache + // Let's fine out + let locals = [ + ("cache", cache.clone()), + ("p1", ptr_1.to_object(py)), + ("p2", ptr_2.to_object(py)), + ] + .into_py_dict(py); + + let pair: PyResult<&PyAny> = py.eval("(cache[p1], cache[p2])", None, Some(locals)); + + match pair { + // the children aren't in the cache, keep drilling down + Err(_) => Ok(Some((ptr_1, ptr_2))), + + // the children are in the cache, create new node & populate cache with it + Ok(tuple) => { + let (p1, p2): (&PyCell, &PyCell) = tuple.extract()?; + add( + py, + cache, + PyNaNode::new( py, - PyNaNode::new( - py, - Some(PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?), - None, - )?, - &ptr, - )?; - Ok(None) - } + Some(PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?), + )?, + &ptr, + )?; + Ok(None) } } } - })?; + } + })?; - self.from_native_to_py_cache(py, &ptr) - } + from_native_to_py_cache(py, cache, &ptr) +} - pub fn py_for_native<'p>( - &'p self, - py: Python<'p>, - ptr: &::Ptr, - allocator: &mut IntAllocator, - ) -> PyResult<&'p PyCell> { - self.from_native_to_py_cache(py, ptr) - .or_else(|err| Ok(self.populate_python(py, ptr, allocator)?)) - } +pub fn py_for_native<'p>( + py: Python<'p>, + cache: &PyObject, + ptr: &::Ptr, + allocator: &mut IntAllocator, +) -> PyResult<&'p PyCell> { + from_native_to_py_cache(py, cache, ptr) + .or_else(|err| Ok(populate_python(py, cache, ptr, allocator)?)) } -pub fn apply_to_tree(mut node: T, mut apply: F) -> PyResult<()> +fn apply_to_tree(mut node: T, mut apply: F) -> PyResult<()> where F: FnMut(T) -> PyResult>, T: Clone, From 0471077cd43020a46343562695708bcdee9c20a6 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 17 Mar 2021 15:18:38 -0700 Subject: [PATCH 30/76] Remove tons of stuff, cargo clippy. --- src/py/api.rs | 116 +----------- src/py/arc_allocator.rs | 143 -------------- src/py/gateway.rs | 8 - src/py/glue.rs | 149 --------------- src/py/int_allocator_gateway.rs | 323 -------------------------------- src/py/mod.rs | 7 - src/py/native_op_lookup.rs | 148 --------------- src/py/native_view.rs | 18 -- src/py/op_fn.rs | 28 +-- src/py/py_int_allocator.rs | 5 +- src/py/py_na_node.rs | 92 +-------- src/py/py_native_mapping.rs | 44 +---- src/py/py_node.rs | 178 ------------------ src/py/run_program.rs | 25 +-- src/py/to_py_node.rs | 7 - 15 files changed, 28 insertions(+), 1263 deletions(-) delete mode 100644 src/py/arc_allocator.rs delete mode 100644 src/py/gateway.rs delete mode 100644 src/py/glue.rs delete mode 100644 src/py/int_allocator_gateway.rs delete mode 100644 src/py/native_op_lookup.rs delete mode 100644 src/py/native_view.rs delete mode 100644 src/py/py_node.rs delete mode 100644 src/py/to_py_node.rs diff --git a/src/py/api.rs b/src/py/api.rs index 8f1ea986..d13ece9e 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -5,98 +5,16 @@ use pyo3::types::{PyBytes, PyDict, PyString}; use pyo3::wrap_pyfunction; use pyo3::PyObject; -//use super::int_allocator_gateway::{PyIntAllocator, PyIntNode}; - use super::py_int_allocator::PyIntAllocator; -use super::py_na_node::{new_cache, PyNaNode}; -use super::py_native_mapping::{add, native_for_py, new_mapping, py_for_native}; +use super::py_na_node::PyNaNode; +use super::py_native_mapping::{native_for_py, new_mapping, py_for_native}; +use super::run_program::{__pyo3_get_function_deserialize_and_run_program, STRICT_MODE}; -//use super::run_program::{__pyo3_get_function_deserialize_and_run_program, STRICT_MODE}; use crate::int_allocator::IntAllocator; -//use crate::py::run_program::OperatorHandlerWithMode; - use crate::cost::Cost; use crate::serialize::{node_from_bytes, node_to_bytes}; -type AllocatorT<'a> = IntAllocator; -//type NodeClass = PyIntNode; - -/* -#[pyclass] -pub struct NativeOpLookup { - nol: usize, // Box>, -} - -#[pymethods] -impl NativeOpLookup { - #[new] - fn new(opcode_lookup_by_name: HashMap>, unknown_op_callback: PyObject) -> Self { - Self::new_from_gnol(Box::new(GenericNativeOpLookup::new( - opcode_lookup_by_name, - unknown_op_callback, - ))) - } -} - -impl Drop for NativeOpLookup { - fn drop(&mut self) { - let _b = - unsafe { Box::from_raw(self.nol as *mut GenericNativeOpLookup) }; - } -} - -impl NativeOpLookup { - fn new_from_gnol(gnol: Box>) -> Self { - NativeOpLookup { - nol: Box::into_raw(gnol) as usize, - } - } -} - -impl NativeOpLookup { - fn gnol<'a, 'p>( - &self, - _py: Python<'p>, - ) -> &'a GenericNativeOpLookup, PyIntNode> { - unsafe { &*(self.nol as *const GenericNativeOpLookup) } - } -} -*/ - -/* -#[pyfunction] -#[allow(clippy::too_many_arguments)] -fn py_run_program( - py: Python, - program: &PyCell, - args: &PyCell, - quote_kw: u8, - apply_kw: u8, - max_cost: Cost, - opcode_lookup_by_name: HashMap>, - py_callback: PyObject, - //op_lookup: Py, - //pre_eval: PyObject, -) -> PyResult<(Cost, PyObject)> { - //let f_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); - //let strict: bool = false; - //let f: Box> = - // Box::new(OperatorHandlerWithMode { f_lookup, strict }); - - _py_run_program( - py, - program, - args, - quote_kw, - apply_kw, - max_cost, - opcode_lookup_by_name, - py_callback, - ) -} -*/ - #[pyfunction] fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult { let ctx: &PyDict = PyDict::new(py); @@ -113,10 +31,6 @@ fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult AllocatorT { - AllocatorT::new() -} - #[pyfunction] fn serialize_from_bytes<'p>(py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyCell> { let py_int_allocator = PyCell::new(py, PyIntAllocator::default())?; @@ -130,8 +44,6 @@ use crate::node::Node; #[pyfunction] fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { - PyNaNode::clear_native_view(sexp, py)?; - let py_int_allocator_cell = PyCell::new(py, PyIntAllocator::default())?; let py_int_allocator: &mut PyIntAllocator = &mut py_int_allocator_cell.borrow_mut(); let allocator: &mut IntAllocator = &mut py_int_allocator.arena; @@ -152,8 +64,8 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(serialize_from_bytes, m)?)?; m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; - //m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; - //m.add("STRICT_MODE", STRICT_MODE)?; + m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; + m.add("STRICT_MODE", STRICT_MODE)?; //m.add_class::()?; // m.add_class::()?; @@ -184,28 +96,15 @@ pub fn py_run_program( opcode_lookup_by_name: HashMap>, py_callback: PyObject, ) -> PyResult<(Cost, PyObject)> { - let arena = PyCell::new( - py, - PyIntAllocator { - arena: IntAllocator::new(), - }, - )?; - let mut arena_ref = arena.borrow_mut(); - let allocator: &mut IntAllocator = &mut arena_ref.arena; - - let arena_as_obj = arena.to_object(py); + let allocator: &mut IntAllocator = &mut IntAllocator::new(); + let cache = new_mapping(py)?; let op_lookup = Box::new(PyOperatorHandler::new( - py, opcode_lookup_by_name, - arena.to_object(py), py_callback, cache.clone(), )?); - PyNaNode::clear_native_view(program, py); let program = native_for_py(py, &cache, program, allocator)?; - - PyNaNode::clear_native_view(args, py); let args = native_for_py(py, &cache, args, allocator)?; let r: Result, EvalErr> = crate::run_program::run_program( @@ -215,7 +114,6 @@ pub fn py_run_program( match r { Ok(reduction) => { let r = py_for_native(py, &cache, &reduction.1, allocator)?; - PyNaNode::clear_native_view(r, py); Ok((reduction.0, r.to_object(py))) } Err(eval_err) => { diff --git a/src/py/arc_allocator.rs b/src/py/arc_allocator.rs deleted file mode 100644 index fc17f510..00000000 --- a/src/py/arc_allocator.rs +++ /dev/null @@ -1,143 +0,0 @@ -use crate::allocator::{Allocator, SExp}; -use crate::err_utils::err; -use crate::reduction::EvalErr; -use std::sync::Arc; - -use lazy_static::*; - -use pyo3::prelude::*; - -#[pyclass(subclass, unsendable)] -#[derive(Clone)] -pub struct ArcAllocator {} - -#[derive(Clone)] -pub struct ArcAtomBuf { - buf: Arc>, - start: u32, - end: u32, -} - -pub enum ArcSExp { - Atom(ArcAtomBuf), - Pair(Arc, Arc), -} - -lazy_static! { - static ref NULL: Arc> = Arc::new(vec![]); - static ref ONE: Arc> = Arc::new(vec![1]); -} - -impl Clone for ArcSExp { - fn clone(&self) -> Self { - match self { - ArcSExp::Atom(a) => Self::Atom(a.clone()), - ArcSExp::Pair(p1, p2) => Self::Pair(p1.clone(), p2.clone()), - } - } -} - -impl ArcAllocator { - pub const fn new() -> Self { - Self {} - } - - pub fn blob(&mut self, v: &str) -> Result> { - let v: Vec = v.into(); - self.new_atom(&v) - } -} - -impl Allocator for ArcAllocator { - type Ptr = ArcSExp; - type AtomBuf = ArcAtomBuf; - - fn new_atom(&mut self, v: &[u8]) -> Result> { - Ok(ArcSExp::Atom(ArcAtomBuf { - buf: Arc::new(v.into()), - start: 0, - end: v.len() as u32, - })) - } - - fn new_pair( - &mut self, - first: Self::Ptr, - rest: Self::Ptr, - ) -> Result> { - Ok(ArcSExp::Pair(Arc::new(first), Arc::new(rest))) - } - - fn new_substr( - &mut self, - node: Self::Ptr, - start: u32, - end: u32, - ) -> Result> { - let atom = match &node { - ArcSExp::Atom(a) => a, - _ => { - return err(node, "substr expected atom, got pair"); - } - }; - let atom_len = atom.end - atom.start; - if start > atom_len { - return err(node, "substr start out of bounds"); - } - if end > atom_len { - return err(node, "substr end out of bounds"); - } - if end < start { - return err(node, "substr invalid bounds"); - } - Ok(ArcSExp::Atom(ArcAtomBuf { - buf: atom.buf.clone(), - start: atom.start, - end: atom.end, - })) - } - - fn atom<'a>(&'a self, node: &'a Self::Ptr) -> &'a [u8] { - match node { - ArcSExp::Atom(a) => &a.buf[a.start as usize..a.end as usize], - _ => panic!("expected atom, got pair"), - } - } - - fn buf<'a>(&'a self, node: &'a Self::AtomBuf) -> &'a [u8] { - &node.buf[node.start as usize..node.end as usize] - } - - fn sexp(&self, node: &ArcSExp) -> SExp { - match node { - ArcSExp::Atom(a) => SExp::Atom(a.clone()), - ArcSExp::Pair(left, right) => { - let p1: &ArcSExp = left; - let p2: &ArcSExp = right; - SExp::Pair(p1.to_owned(), p2.to_owned()) - } - } - } - - fn null(&self) -> ArcSExp { - ArcSExp::Atom(ArcAtomBuf { - buf: NULL.to_owned(), - start: 0, - end: 0, - }) - } - - fn one(&self) -> ArcSExp { - ArcSExp::Atom(ArcAtomBuf { - buf: ONE.to_owned(), - start: 0, - end: 1, - }) - } -} - -impl Default for ArcAllocator { - fn default() -> Self { - Self::new() - } -} diff --git a/src/py/gateway.rs b/src/py/gateway.rs deleted file mode 100644 index 6093f5ee..00000000 --- a/src/py/gateway.rs +++ /dev/null @@ -1,8 +0,0 @@ -use pyo3::prelude::{PyObject, PyResult, Python}; - -use crate::allocator::Allocator; - -pub trait PythonGateway { - fn to_pyobject(self, py: Python, ptr: ::Ptr) -> PyResult; - fn from_pyobject(self, py: Python, o: PyObject) -> PyResult<::Ptr>; -} diff --git a/src/py/glue.rs b/src/py/glue.rs deleted file mode 100644 index 7e245d8a..00000000 --- a/src/py/glue.rs +++ /dev/null @@ -1,149 +0,0 @@ -use pyo3::prelude::*; -use pyo3::types::{PyBytes, PyDict, PyString}; -use pyo3::PyClass; -use pyo3::PyObject; - -use super::arc_allocator::ArcAllocator; -use super::native_op_lookup::GenericNativeOpLookup; -use super::py_node::PyNode; -use super::to_py_node::ToPyNode; - -use crate::allocator::Allocator; -use crate::cost::Cost; -use crate::node::Node; -use crate::reduction::{EvalErr, Reduction}; -use crate::run_program::{run_program, PostEval, PreEval}; -use crate::serialize::{node_from_bytes, node_to_bytes}; - -impl ToPyNode for ArcAllocator { - fn to_pynode(&self, ptr: &Self::Ptr) -> PyNode { - PyNode::new(ptr.clone()) - } -} - -fn note_result(obj: &PyObject, result: Option<&T>) -where - T: ToPyObject, -{ - Python::with_gil(|py| { - if let Some(node) = result { - let node: PyObject = node.to_object(py); - let _r: PyResult = obj.call1(py, (node,)); - } - }); -} - -fn post_eval_for_pyobject(py: Python, obj: PyObject) -> Option>> -where - A::Ptr: ToPyObject, -{ - let py_post_eval: Option>> = if obj.is_none(py) { - None - } else { - Some(Box::new(move |result: Option<&A::Ptr>| { - note_result(&obj, result) - })) - }; - - py_post_eval -} - -#[allow(clippy::too_many_arguments)] -pub fn _py_run_program<'p, 'a, 'n, A, N>( - py: Python<'p>, - allocator: &'a mut A, - program: &'n N, - args: &'n N, - quote_kw: u8, - apply_kw: u8, - max_cost: Cost, - op_lookup: Box>, - pre_eval: PyObject, -) -> PyResult<(Cost, N)> -where - A: 'static + Allocator + ToPyNode, - N: 'static + PyClass + IntoPy + Clone, - ::Ptr: IntoPy + From<&'n N> + From + ToPyObject, -{ - let py_pre_eval_t: Option> = if pre_eval.is_none(py) { - None - } else { - Some(Box::new(move |allocator, program, args| { - Python::with_gil(|py| { - let program_clone: N = allocator.to_pynode(program); - let args: N = allocator.to_pynode(args); - let r: PyResult = pre_eval.call1(py, (program_clone, args)); - match r { - Ok(py_post_eval) => Ok(post_eval_for_pyobject::(py, py_post_eval)), - Err(ref err) => { - let program: Node = Node::new(allocator, program.clone()); - program.err(&err.to_string()) - } - } - }) - })) - }; - - let r: Result::Ptr>, EvalErr<::Ptr>> = run_program( - allocator, - &program.into(), - &args.into(), - quote_kw, - apply_kw, - max_cost, - op_lookup, - py_pre_eval_t, - ); - match r { - Ok(reduction) => Ok((reduction.0, allocator.to_pynode(&reduction.1))), - Err(eval_err) => { - let node: PyObject = eval_err.0.to_object(py); - let s: String = eval_err.1; - let s1: &str = &s; - let msg: &PyString = PyString::new(py, s1); - match raise_eval_error(py, msg, node) { - Err(x) => Err(x), - _ => panic!(), - } - } - } -} - -fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult { - let ctx: &PyDict = PyDict::new(py); - ctx.set_item("msg", msg)?; - ctx.set_item("sexp", sexp)?; - let r = py.run( - "from clvm.EvalError import EvalError; raise EvalError(msg, sexp)", - None, - Some(ctx), - ); - match r { - Err(x) => Err(x), - Ok(_) => Ok(ctx.into()), - } -} - -pub fn _serialize_from_bytes(allocator: &mut A, blob: &[u8]) -> N -where - A: ToPyNode, -{ - let n = node_from_bytes(allocator, blob).unwrap(); - allocator.to_pynode(&n) -} - -pub fn _serialize_to_bytes( - allocator: &A, - py: Python, - sexp: &PyAny, -) -> PyResult -where - N: PyClass + Clone, - ::Ptr: From, -{ - let py_node: N = sexp.extract()?; - let node_t: Node = Node::new(allocator, py_node.into()); - let blob = node_to_bytes(&node_t).unwrap(); - let pybytes = PyBytes::new(py, &blob); - Ok(pybytes.to_object(py)) -} diff --git a/src/py/int_allocator_gateway.rs b/src/py/int_allocator_gateway.rs deleted file mode 100644 index 79b0badb..00000000 --- a/src/py/int_allocator_gateway.rs +++ /dev/null @@ -1,323 +0,0 @@ -use std::cell::{Cell, Ref, RefCell}; - -use pyo3::prelude::*; -use pyo3::types::PyBytes; -use pyo3::types::PyTuple; -use pyo3::types::PyType; - -use crate::allocator::{Allocator, SExp}; -use crate::int_allocator::IntAllocator; - -#[pyclass(subclass, unsendable)] -pub struct PyIntAllocator { - pub arena: IntAllocator, -} - -pub struct PyView { - atom: PyObject, - pair: PyObject, -} - -impl PyView { - pub fn new(atom: &PyObject, pair: &PyObject) -> Self { - let atom = atom.clone(); - let pair = pair.clone(); - PyView { atom, pair } - } - - fn py_bytes<'p>(&'p self, py: Python<'p>) -> Option<&'p PyBytes> { - // this glue returns a &[u8] if self.atom has PyBytes behind it - let r: Option<&PyBytes> = self.atom.extract(py).ok(); - r - } - - fn py_pair<'p>( - &'p self, - py: Python<'p>, - ) -> Option<(&'p PyCell, &'p PyCell)> { - let args: &PyTuple = self.pair.extract(py).ok()?; - let p0: &'p PyCell = args.get_item(0).extract().unwrap(); - let p1: &'p PyCell = args.get_item(1).extract().unwrap(); - Some((p0, p1)) - } -} - -#[pyclass(subclass, unsendable)] -pub struct PyIntNode { - pub arena: PyObject, // &PyCell - // rust view - pub native_view: Cell::Ptr>>, - // python view - pub py_view: RefCell>, -} - -impl PyIntNode { - pub fn new(arena: PyObject, native_view: Option, py_view: Option) -> Self { - let native_view = Cell::new(native_view); - let py_view = RefCell::new(py_view); - Self { - arena, - native_view, - py_view, - } - } - - pub fn from_ptr<'p>( - py: Python<'p>, - py_int_allocator: PyObject, - ptr: ::Ptr, - ) -> PyResult<&'p PyCell> { - let py_int_node = PyCell::new(py, PyIntNode::new(py_int_allocator, Some(ptr), None)); - py_int_node - } - - fn allocator<'p>(&'p self, py: Python<'p>) -> PyResult> { - let allocator: &PyCell = self.arena.extract(py)?; - Ok(allocator.try_borrow()?) - } - - fn allocator_mut<'p>(&'p self, py: Python<'p>) -> PyResult> { - let allocator: &PyCell = self.arena.extract(py)?; - Ok(allocator.try_borrow_mut()?) - } - - pub fn ptr( - slf: &PyCell, - py: Option, - arena: PyObject, - allocator: &mut IntAllocator, - ) -> ::Ptr { - if let Some(r) = slf.borrow().native_view.get() { - r - } else { - if let Some(py) = py { - let p = slf.borrow(); - let mut to_cast: Vec = vec![slf.to_object(py)]; - - Self::ensure_native_view(to_cast, arena, allocator, py); - slf.borrow().native_view.get().unwrap() - } else { - panic!("can't cast from python to native") - } - } - } - - /* - pub fn get_py_view<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult>> { - let t0: PyRef = slf.borrow(); - { - let t1: Ref> = t0.py_view.borrow(); - if t1.is_none() { - let mut t2: PyRefMut = t0.allocator_mut(py)?; - let allocator: &mut IntAllocator = &mut t2.arena; - Self::ensure_python_view(vec![slf.to_object(py)], allocator, py)?; - } - } - Ok(t0.py_view.borrow()) - } - */ - - pub fn ensure_native_view( - mut to_cast: Vec, - arena: PyObject, - allocator: &mut IntAllocator, - py: Python, - ) { - loop { - let t: Option = to_cast.pop(); - match t { - None => break, - Some(t0) => { - let t0_5: &PyAny = t0.extract(py).unwrap(); - let t1: &PyCell = t0_5.downcast().unwrap(); - let mut t2: PyRefMut = t1.borrow_mut(); - if t2.native_view.get().is_none() { - let py_view_ref: Ref> = t2.py_view.borrow(); - let py_view = py_view_ref.as_ref().unwrap(); - match py_view.py_bytes(py) { - Some(blob) => { - let new_ptr = allocator.new_atom(blob.as_bytes()).unwrap(); - t2.native_view.set(Some(new_ptr)); - } - None => { - let (p1, p2) = py_view.py_pair(py).unwrap(); - // check if both p1 and p2 have native views - // if so build and cache native view for t - let r1: Option<::Ptr> = - p1.borrow().native_view.get(); - let r2: Option<::Ptr> = - p2.borrow().native_view.get(); - if let (Some(s1), Some(s2)) = (r1, r2) { - let ptr = allocator.new_pair(s1, s2).unwrap(); - t2.native_view.set(Some(ptr)); - t2.arena = arena.clone(); - } else { - // otherwise, push t, push p1, push p2 back on stack to be processed - to_cast.push(p1.to_object(py)); - to_cast.push(p2.to_object(py)); - } - } - } - } - } - } - } - } - - pub fn ensure_python_view( - mut to_cast: Vec, - allocator: &mut IntAllocator, - py: Python, - ) -> PyResult<()> { - loop { - let t = to_cast.pop(); - match t { - None => break, - Some(t0) => { - let t1: &PyAny = t0.extract(py).unwrap(); - let t2: &PyCell = t1.downcast().unwrap(); - let t3: PyRef = t2.borrow(); - - if t3.py_view.borrow().is_some() { - continue; - } - let ptr = t3.native_view.get().unwrap(); - match allocator.sexp(&ptr) { - SExp::Atom(a) => { - let as_u8: &[u8] = allocator.buf(&a); - let py_bytes = PyBytes::new(py, as_u8); - let py_object: PyObject = py_bytes.to_object(py); - let py_view = PyView { - atom: py_object, - pair: ().to_object(py), - }; - t3.py_view.replace(Some(py_view)); - } - SExp::Pair(p1, p2) => { - // create new n1, n2 child nodes of t - let arena = t3.arena.clone(); - let native_view = Cell::new(Some(p1)); - let py_view = RefCell::new(None); - let n1 = PyCell::new( - py, - PyIntNode { - arena, - native_view, - py_view, - }, - )?; - let arena = t3.arena.clone(); - let native_view = Cell::new(Some(p2)); - let py_view = RefCell::new(None); - let n2 = PyCell::new( - py, - PyIntNode { - arena, - native_view, - py_view, - }, - )?; - let py_object = PyTuple::new(py, &[n1, n2]); - let py_view = PyView { - pair: py_object.to_object(py), - atom: ().to_object(py), - }; - t3.py_view.replace(Some(py_view)); - to_cast.push(n1.to_object(py)); - to_cast.push(n2.to_object(py)); - } - } - } - } - } - Ok(()) - } -} - -#[pymethods] -impl PyIntNode { - #[classmethod] - fn new_atom(cls: &PyType, py: Python, atom: &PyBytes) -> PyResult { - let none: PyObject = py.None(); - let py_view = Some(PyView::new(&atom.to_object(py), &none)); - Ok(PyIntNode::new(none, None, py_view)) - } - - #[classmethod] - fn new_pair( - cls: &PyType, - py: Python, - p1: &PyCell, - p2: &PyCell, - ) -> PyResult { - let pair: &PyTuple = PyTuple::new(py, &[p1, p2]); - let none: PyObject = py.None(); - // TODO: ensure `pair` is a tuple of two `PyIntNode` - let py_view = Some(PyView::new(&none, &pair.to_object(py))); - Ok(PyIntNode::new(none, None, py_view)) - } - - #[getter(arena)] - pub fn get_arena(&self) -> PyObject { - self.arena.clone() - } - - #[getter(pair)] - pub fn pair<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - let t0: PyRef = slf.borrow(); - let t1: Ref> = t0.py_view.borrow(); - if t1.is_none() { - let mut t2: PyRefMut = t0.allocator_mut(py)?; - let allocator: &mut IntAllocator = &mut t2.arena; - Self::ensure_python_view(vec![slf.to_object(py)], allocator, py)?; - } - let t3 = &t1.as_ref().unwrap().pair; - Ok(t3.clone()) - - /* - let allocator = self.allocator(py)?; - let allocator: &IntAllocator = &allocator.arena; - match allocator.sexp(&self.ptr) { - SExp::Pair(p1, p2) => { - let v: &PyTuple = PyTuple::new(py, &[p1, p2]); - let v: PyObject = v.into(); - Ok(Some(v)) - } - _ => Ok(None), - }*/ - } - - /* - pub fn _pair(&self) -> Option<(PyNode, PyNode)> { - match ArcAllocator::new().sexp(&self.node) { - SExp::Pair(p1, p2) => Some((p1.into(), p2.into())), - _ => None, - } - } - */ - #[getter(atom)] - pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - let t0: PyRef = slf.borrow(); - let t1: Ref> = t0.py_view.borrow(); - if t1.is_none() { - let mut t2: PyRefMut = t0.allocator_mut(py)?; - let allocator: &mut IntAllocator = &mut t2.arena; - Self::ensure_python_view(vec![slf.to_object(py)], allocator, py)?; - } - let t3 = &t1.as_ref().unwrap().atom; - Ok(t3.clone()) - /* - let allocator = self.allocator(py)?; - let allocator: &IntAllocator = &allocator.arena; - match allocator.sexp(&self.ptr) { - SExp::Atom(atom) => { - let s: &[u8] = allocator.buf(&atom); - let s: &PyBytes = PyBytes::new(py, s); - let s: PyObject = s.into(); - Ok(Some(s)) - } - _ => Ok(None), - } - */ - } -} diff --git a/src/py/mod.rs b/src/py/mod.rs index 9ce091a1..2eabf6f6 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -1,15 +1,8 @@ pub mod api; -pub mod arc_allocator; pub mod f_table; -pub mod glue; -//pub mod int_allocator_gateway; -pub mod native_op_lookup; -//pub mod native_view; pub mod op_fn; pub mod py_int_allocator; pub mod py_na_node; pub mod py_native_mapping; -pub mod py_node; pub mod py_view; pub mod run_program; -pub mod to_py_node; diff --git a/src/py/native_op_lookup.rs b/src/py/native_op_lookup.rs deleted file mode 100644 index 73d537a5..00000000 --- a/src/py/native_op_lookup.rs +++ /dev/null @@ -1,148 +0,0 @@ -use std::collections::HashMap; -use std::marker::PhantomData; - -use pyo3::prelude::*; -use pyo3::types::{PyString, PyTuple}; -use pyo3::PyClass; - -use crate::allocator::Allocator; -use crate::cost::Cost; -use crate::reduction::{EvalErr, Reduction, Response}; -use crate::run_program::OperatorHandler; - -use super::f_table::{f_lookup_for_hashmap, FLookup}; - -use super::to_py_node::ToPyNode; - -/// turn a `PyErr` into an `EvalErr

` if at all possible -/// otherwise, return a `PyErr` - -fn eval_err_for_pyerr<'s, 'p: 's, 'e: 's, P, N>( - py: Python<'p>, - pyerr: &'e PyErr, -) -> PyResult> -where - P: From, - N: FromPyObject<'s>, -{ - let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; - let arg0: &PyString = args.get_item(0).extract()?; - let sexp: N = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: P = sexp.into(); - let s: String = arg0.to_str()?.to_string(); - Ok(EvalErr(node, s)) -} -#[derive(Clone)] -pub struct GenericNativeOpLookup -where - A: Allocator + ToPyNode, - N: PyClass, - ::Ptr: From, -{ - py_callback: PyObject, - f_lookup: FLookup, - phantom_data: PhantomData, -} - -impl GenericNativeOpLookup -where - A: Allocator + ToPyNode, - N: PyClass, - ::Ptr: From, -{ - pub fn new( - opcode_lookup_by_name: HashMap>, - unknown_op_callback: PyObject, - ) -> Self { - let f_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); - - Self { - py_callback: unknown_op_callback, - f_lookup, - phantom_data: PhantomData, - } - } -} - -impl OperatorHandler for GenericNativeOpLookup -where - A: Allocator + ToPyNode, - N: PyClass + Clone + IntoPy, - ::Ptr: From, -{ - fn op( - &self, - allocator: &mut A, - op: A::AtomBuf, - argument_list: &::Ptr, - max_cost: Cost, - ) -> Response<::Ptr> { - eval_op::( - &self.f_lookup, - &self.py_callback, - allocator, - &op, - argument_list, - max_cost, - ) - } -} - -fn eval_op( - f_lookup: &FLookup, - py_callback: &PyObject, - allocator: &mut A, - o: &::AtomBuf, - argument_list: &::Ptr, - max_cost: Cost, -) -> Response<::Ptr> -where - A: Allocator + ToPyNode, - ::Ptr: From, - N: PyClass + Clone, - N: IntoPy, -{ - let op = allocator.buf(o); - if op.len() == 1 { - if let Some(f) = f_lookup[op[0] as usize] { - return f(allocator, argument_list.clone(), max_cost); - } - } - - Python::with_gil(|py| { - let pynode: N = allocator.to_pynode(argument_list); - let r1 = py_callback.call1(py, (op, pynode)); - match r1 { - Err(pyerr) => { - let eval_err: PyResult::Ptr>> = - eval_err_for_pyerr(py, &pyerr); - let r: EvalErr<::Ptr> = - unwrap_or_eval_err(eval_err, argument_list, "unexpected exception")?; - Err(r) - } - Ok(o) => { - let pair: &PyTuple = - unwrap_or_eval_err(o.extract(py), argument_list, "expected tuple")?; - - let i0: u32 = - unwrap_or_eval_err(pair.get_item(0).extract(), argument_list, "expected u32")?; - - let py_node: N = - unwrap_or_eval_err(pair.get_item(1).extract(), argument_list, "expected node")?; - - let node: ::Ptr = py_node.into(); - Ok(Reduction(i0 as Cost, node)) - } - } - }) -} - -fn unwrap_or_eval_err(obj: PyResult, err_node: &P, msg: &str) -> Result> -where - P: Clone, -{ - match obj { - Err(_py_err) => Err(EvalErr(err_node.clone(), msg.to_string())), - Ok(o) => Ok(o), - } -} diff --git a/src/py/native_view.rs b/src/py/native_view.rs deleted file mode 100644 index 01ecb94f..00000000 --- a/src/py/native_view.rs +++ /dev/null @@ -1,18 +0,0 @@ -/* -use pyo3::prelude::PyObject; - -use crate::allocator::Allocator; -use crate::int_allocator::IntAllocator; - -#[derive(Clone)] -pub struct NativeView { - pub arena: PyObject, // PyCell, - pub ptr: ::Ptr, -} - -impl NativeView { - pub fn new(arena: PyObject, ptr: ::Ptr) -> Self { - NativeView { arena, ptr } - } -} -*/ \ No newline at end of file diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 152057b5..d457e58c 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -1,4 +1,3 @@ -use std::cell::RefCell; use std::collections::HashMap; use pyo3::types::PyBytes; @@ -7,7 +6,6 @@ use pyo3::types::PyTuple; use pyo3::PyCell; use pyo3::PyErr; use pyo3::PyObject; -use pyo3::PyRefMut; use pyo3::PyResult; use pyo3::Python; use pyo3::ToPyObject; @@ -24,23 +22,19 @@ use super::py_native_mapping::{native_for_py, py_for_native}; pub struct PyOperatorHandler { native_lookup: FLookup, - arena: PyObject, py_callable: PyObject, pub cache: PyObject, } impl PyOperatorHandler { pub fn new( - py: Python, opcode_lookup_by_name: HashMap>, - arena: PyObject, py_callable: PyObject, cache: PyObject, ) -> PyResult { let native_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); Ok(PyOperatorHandler { native_lookup, - arena, py_callable, cache, }) @@ -51,7 +45,6 @@ impl PyOperatorHandler { pub fn invoke_py_obj( &self, obj: PyObject, - arena: PyObject, allocator: &mut IntAllocator, op_buf: ::AtomBuf, args: &::Ptr, @@ -64,16 +57,11 @@ impl PyOperatorHandler { args, "can't uncache", )?; - let r1 = obj.call1(py, (op, r.to_object(py))); + let r1 = obj.call1(py, (op, r.to_object(py), max_cost)); match r1 { Err(pyerr) => { - let eval_err: PyResult> = eval_err_for_pyerr( - py, - &pyerr, - self.cache.clone(), - arena.clone(), - allocator, - ); + let eval_err: PyResult> = + eval_err_for_pyerr(py, &pyerr, self.cache.clone(), allocator); let r: EvalErr = unwrap_or_eval_err(eval_err, args, "unexpected exception")?; Err(r) @@ -111,14 +99,7 @@ impl OperatorHandler for PyOperatorHandler { } } - self.invoke_py_obj( - self.py_callable.clone(), - self.arena.clone(), - allocator, - op_buf, - args, - max_cost, - ) + self.invoke_py_obj(self.py_callable.clone(), allocator, op_buf, args, max_cost) } } @@ -128,7 +109,6 @@ fn eval_err_for_pyerr<'p>( py: Python<'p>, pyerr: &PyErr, cache: PyObject, - arena: PyObject, allocator: &mut IntAllocator, ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs index 80a87c76..0b5c53f3 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_int_allocator.rs @@ -1,7 +1,4 @@ -use pyo3::basic::CompareOp; -use pyo3::prelude::*; - -use pyo3::PyObjectProtocol; +use pyo3::prelude::pyclass; use crate::int_allocator::IntAllocator; diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 80507551..13e0569e 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -1,97 +1,16 @@ -use std::cell::Cell; - use pyo3::prelude::*; -use pyo3::types::{IntoPyDict, PyBytes, PyTuple, PyType}; - -use crate::allocator::{Allocator, SExp}; -use crate::int_allocator::IntAllocator; +use pyo3::types::{PyBytes, PyTuple, PyType}; -use super::py_int_allocator::PyIntAllocator; use super::py_view::PyView; #[pyclass(weakref, subclass)] pub struct PyNaNode { pub py_view: Option, - pub int_cache: Cell>, - //int_arena_cache: PyObject, // WeakKeyDict[PyIntAllocator, int] -} - -pub fn new_cache(py: Python) -> PyResult { - Ok(py - .eval("__import__('weakref').WeakValueDictionary()", None, None)? - .to_object(py)) -} - -pub fn add_to_cache( - py: Python, - cache: &PyObject, - ptr: ::Ptr, - value: &PyCell, -) -> PyResult<()> { - //return Ok(()); - let locals = [ - ("cache", cache.clone()), - ("key", ptr.to_object(py)), - ("value", value.to_object(py)), - ] - .into_py_dict(py); - - Ok(py.run("cache[key] = value", None, Some(locals))?) -} - -pub fn from_cache( - py: Python, - cache: &PyObject, - ptr: &::Ptr, -) -> PyResult> { - let locals = [("cache", cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); - py.eval("cache.get(key)", None, Some(locals))?.extract() -} - -pub fn apply_to_tree(mut node: T, mut apply: F) -> PyResult<()> -where - F: FnMut(T) -> PyResult>, - T: Clone, -{ - let mut items = vec![node]; - loop { - let t = items.pop(); - if let Some(obj) = t { - if let Some((p0, p1)) = apply(obj.clone())? { - items.push(obj); - items.push(p0); - items.push(p1); - } - } else { - break; - } - } - Ok(()) } impl PyNaNode { pub fn new(py: Python, py_view: Option) -> PyResult<&PyCell> { - let int_cache = Cell::new(None); - PyCell::new(py, PyNaNode { py_view, int_cache }) - } - - pub fn clear_native_view(slf: &PyCell, py: Python) -> PyResult<()> { - apply_to_tree(slf.to_object(py), move |obj: PyObject| { - let mut node: PyRefMut = obj.extract(py)?; - assert!(node.py_view.is_some()); - Ok(if let Some(PyView::Pair(tuple)) = &node.py_view { - let (p0, p1): (PyObject, PyObject) = tuple.extract(py)?; - if node.int_cache.get().is_some() { - node.int_cache.set(None); - Some((p0, p1)) - } else { - None - } - } else { - node.int_cache.set(None); - None - }) - }) + PyCell::new(py, PyNaNode { py_view }) } } @@ -103,14 +22,12 @@ impl PyNaNode { let py_view = PyView::new_pair(py, tuple)?; Self { py_view: Some(py_view), - int_cache: Cell::new(None), } } else { let py_bytes: &PyBytes = obj.extract()?; let py_view = PyView::new_atom(py, py_bytes); Self { py_view: Some(py_view), - int_cache: Cell::new(None), } }) } @@ -155,11 +72,6 @@ impl PyNaNode { } } - #[getter(native)] - pub fn native<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - Ok(slf.borrow().int_cache.get().to_object(py)) - } - #[getter(python)] pub fn python<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { Ok(match &slf.borrow().py_view { diff --git a/src/py/py_native_mapping.rs b/src/py/py_native_mapping.rs index 17eb3958..d070850c 100644 --- a/src/py/py_native_mapping.rs +++ b/src/py/py_native_mapping.rs @@ -1,45 +1,15 @@ use pyo3::prelude::*; -//use pyo3::pyclass::PyClassAlloc;` -use pyo3::types::{IntoPyDict, PyBytes, PyTuple, PyType}; +use pyo3::types::{IntoPyDict, PyBytes, PyTuple}; use crate::allocator::{Allocator, SExp}; use crate::int_allocator::IntAllocator; -use super::py_int_allocator::PyIntAllocator; use super::py_view::PyView; use super::py_na_node::PyNaNode; -/* -pub trait PyNativeMapping { - fn add( - &self, - py: Python, - obj: &PyCell, - ptr: &::Ptr, - ) -> PyResult<()>; - - fn native_for_py( - &self, - py: Python, - obj: &PyCell, - allocator: &mut IntAllocator, - ) -> PyResult<::Ptr>; - - fn py_for_native<'p>( - &'p self, - py: Python<'p>, - ptr: &::Ptr, - allocator: &mut IntAllocator, - ) -> PyResult<&'p PyCell>; -} -*/ - pub fn new_mapping(py: Python) -> PyResult { - Ok(py - //.eval("__import__('weakref').WeakValueDictionary()", None, None)? - .eval("dict()", None, None)? - .to_object(py)) + Ok(py.eval("dict()", None, None)?.to_object(py)) } pub fn add( @@ -48,8 +18,6 @@ pub fn add( obj: &PyCell, ptr: &::Ptr, ) -> PyResult<()> { - //obj.borrow().int_cache.set(Some(ptr.clone())); - let locals = [ ("cache", cache.clone()), ("obj", obj.to_object(py)), @@ -125,7 +93,7 @@ pub fn native_for_py( allocator: &mut IntAllocator, ) -> PyResult<::Ptr> { from_py_to_native_cache(py, cache, obj) - .or_else(|err| populate_native(py, cache, obj, allocator)) + .or_else(|_err| populate_native(py, cache, obj, allocator)) } // native to py methods @@ -145,7 +113,7 @@ fn populate_python<'p>( ptr: &::Ptr, allocator: &mut IntAllocator, ) -> PyResult<&'p PyCell> { - apply_to_tree(ptr.clone(), move |ptr| { + apply_to_tree(*ptr, move |ptr| { // is it in cache yet? if from_native_to_py_cache(py, cache, &ptr).is_ok() { // yep, we're done @@ -212,10 +180,10 @@ pub fn py_for_native<'p>( allocator: &mut IntAllocator, ) -> PyResult<&'p PyCell> { from_native_to_py_cache(py, cache, ptr) - .or_else(|err| Ok(populate_python(py, cache, ptr, allocator)?)) + .or_else(|_err| Ok(populate_python(py, cache, ptr, allocator)?)) } -fn apply_to_tree(mut node: T, mut apply: F) -> PyResult<()> +fn apply_to_tree(node: T, mut apply: F) -> PyResult<()> where F: FnMut(T) -> PyResult>, T: Clone, diff --git a/src/py/py_node.rs b/src/py/py_node.rs deleted file mode 100644 index 52b2fdd0..00000000 --- a/src/py/py_node.rs +++ /dev/null @@ -1,178 +0,0 @@ -use super::arc_allocator::{ArcAllocator, ArcSExp}; -use crate::allocator::{Allocator, SExp}; -use crate::reduction::EvalErr; -use std::cell::RefCell; - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::{PyBytes, PyTuple}; - -#[pyclass(subclass, unsendable)] -#[derive(Clone)] -pub struct PyNode { - node: ArcSExp, - pyobj: RefCell>, -} - -impl std::convert::From> for pyo3::PyErr { - fn from(v: EvalErr) -> Self { - PyValueError::new_err(v.0) - } -} - -fn extract_atom(allocator: &mut ArcAllocator, obj: &PyAny) -> PyResult { - let py_bytes: &PyBytes = obj.extract()?; - let r: &[u8] = obj.extract()?; - let inner_node = allocator.new_atom(r)?; - let py_node = PyNode::new_cached(inner_node, Some(py_bytes.into())); - Ok(py_node) -} - -fn extract_node<'a>(_allocator: &ArcAllocator, obj: &'a PyAny) -> PyResult> { - let ps: &PyCell = obj.downcast()?; - let node: PyRef<'a, PyNode> = ps.try_borrow()?; - Ok(node) -} - -fn extract_tuple(allocator: &mut ArcAllocator, obj: &PyAny) -> PyResult { - let v: &PyTuple = obj.downcast()?; - if v.len() != 2 { - return Err(PyValueError::new_err("SExp tuples must be size 2")); - } - let i0: &PyAny = v.get_item(0); - let i1: &PyAny = v.get_item(1); - let left: PyRef = extract_node(allocator, i0)?; - let right: PyRef = extract_node(allocator, i1)?; - let left: &PyNode = &left; - let right: &PyNode = &right; - let left: ArcSExp = left.into(); - let right: ArcSExp = right.into(); - let node: ArcSExp = allocator.new_pair(left, right)?; - let py_node = PyNode::new_cached(node, Some(obj.into())); - - Ok(py_node) -} - -impl From<&ArcSExp> for PyNode { - fn from(item: &ArcSExp) -> Self { - item.clone().into() - } -} - -impl From for PyNode { - fn from(item: ArcSExp) -> Self { - Self::new(item) - } -} - -impl<'source> FromPyObject<'source> for ArcSExp { - fn extract(obj: &'source PyAny) -> PyResult { - let py_node: PyNode = obj.extract()?; - Ok(py_node.into()) - } -} - -impl ToPyObject for ArcSExp { - fn to_object(&self, py: Python<'_>) -> PyObject { - let pynode: PyNode = self.into(); - let pynode: &PyCell = PyCell::new(py, pynode).unwrap(); - let pa: &PyAny = pynode; - pa.to_object(py) - } -} - -#[pymethods] -impl PyNode { - #[new] - pub fn py_new(obj: &PyAny) -> PyResult { - let mut allocator = ArcAllocator::new(); - let node: Self = { - let n = extract_atom(&mut allocator, obj); - if let Ok(r) = n { - r - } else { - extract_tuple(&mut allocator, obj)? - } - }; - Ok(node) - } - - #[getter(pair)] - pub fn pair(&self, py: Python) -> PyResult> { - match ArcAllocator::new().sexp(&self.node) { - SExp::Pair(p1, p2) => { - { - let mut borrowed_pair = self.pyobj.borrow_mut(); - if borrowed_pair.is_none() { - let r1 = PyCell::new(py, Self::new(p1))?; - let r2 = PyCell::new(py, Self::new(p2))?; - let v: &PyTuple = PyTuple::new(py, &[r1, r2]); - let v: PyObject = v.into(); - *borrowed_pair = Some(v); - } - }; - Ok(self.pyobj.borrow().clone()) - } - _ => Ok(None), - } - } - - pub fn _pair(&self) -> Option<(Self, Self)> { - match ArcAllocator::new().sexp(&self.node) { - SExp::Pair(p1, p2) => Some((p1.into(), p2.into())), - _ => None, - } - } - - #[getter(atom)] - pub fn atom(&self, py: Python) -> Option { - let alloc = ArcAllocator::new(); - match alloc.sexp(&self.node) { - SExp::Atom(atom) => { - { - let mut borrowed_bytes = self.pyobj.borrow_mut(); - if borrowed_bytes.is_none() { - let b: &PyBytes = PyBytes::new(py, alloc.buf(&atom)); - let obj: PyObject = b.into(); - *borrowed_bytes = Some(obj); - }; - } - self.pyobj.borrow().clone() - } - _ => None, - } - } -} - -impl PyNode { - pub const fn new(node: ArcSExp) -> Self { - Self::new_cached(node, None) - } - - pub const fn new_cached(node: ArcSExp, py_val: Option) -> Self { - Self { - node, - pyobj: RefCell::new(py_val), - } - } -} - -impl From<&PyNode> for ArcSExp { - fn from(node: &PyNode) -> Self { - node.clone().into() - } -} - -impl From for ArcSExp { - fn from(node: PyNode) -> Self { - node.node - } -} - -impl IntoPy for ArcSExp { - fn into_py(self, py: Python<'_>) -> PyObject { - let pynode: PyNode = self.into(); - let pynode: &PyCell = PyCell::new(py, pynode).unwrap(); - pynode.to_object(py) - } -} diff --git a/src/py/run_program.rs b/src/py/run_program.rs index 71d1cdb5..271e358d 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -1,5 +1,3 @@ -/* -use std::cell::{Cell, RefCell}; use std::collections::HashMap; use crate::allocator::Allocator; @@ -8,12 +6,13 @@ use crate::err_utils::err; use crate::int_allocator::IntAllocator; use crate::more_ops::op_unknown; use crate::node::Node; -use crate::py::f_table::{f_lookup_for_hashmap, FLookup}; -use crate::py::py_int_allocator::{PyIntAllocator}; use crate::reduction::Response; use crate::run_program::{run_program, OperatorHandler}; use crate::serialize::{node_from_bytes, node_to_bytes, serialized_length_from_bytes}; +use super::f_table::{f_lookup_for_hashmap, FLookup}; +use super::py_native_mapping::{new_mapping, py_for_native}; + use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict}; @@ -67,6 +66,7 @@ pub fn deserialize_and_run_program( Box::new(OperatorHandlerWithMode { f_lookup, strict }); let program = node_from_bytes(&mut allocator, program)?; let args = node_from_bytes(&mut allocator, args)?; + let cache = new_mapping(py)?; let r = py.allow_threads(|| { run_program( @@ -81,18 +81,10 @@ pub fn deserialize_and_run_program( ) }); match r { - Ok(reduction) => { - let py_int_allocator = PyCell::new(py, PyIntAllocator { arena: allocator })?; - let py_int_node = PyCell::new( - py, - PyIntNode { - arena: py_int_allocator.to_object(py), - py_view: RefCell::new(None), - native_view: Cell::new(Some(reduction.1)), - }, - )?; - Ok((reduction.0, py_int_node.to_object(py))) - } + Ok(reduction) => Ok(( + reduction.0, + py_for_native(py, &cache, &reduction.1, &mut allocator)?.to_object(py), + )), Err(eval_err) => { let node_as_blob = node_to_bytes(&Node::new(&allocator, eval_err.0))?; let msg = eval_err.1; @@ -122,4 +114,3 @@ raise EvalError(msg, sexp)", pub fn serialized_length(program: &[u8]) -> PyResult { Ok(serialized_length_from_bytes(program)?) } -*/ diff --git a/src/py/to_py_node.rs b/src/py/to_py_node.rs deleted file mode 100644 index 747fc91c..00000000 --- a/src/py/to_py_node.rs +++ /dev/null @@ -1,7 +0,0 @@ -use pyo3::PyClass; - -use crate::allocator::Allocator; - -pub trait ToPyNode: Allocator { - fn to_pynode(&self, ptr: &Self::Ptr) -> N; -} From 7f78bb124dfab09a1ef52c1ee34a1c821fe13a82 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 17 Mar 2021 15:57:18 -0700 Subject: [PATCH 31/76] Rename `PyNaNode` to `PyNode`. --- src/py/api.rs | 12 ++++++------ src/py/op_fn.rs | 6 +++--- src/py/py_na_node.rs | 12 ++++++------ src/py/py_native_mapping.rs | 28 ++++++++++++++-------------- src/py/py_view.rs | 6 +++--- src/py1/mod.rs | 8 -------- 6 files changed, 32 insertions(+), 40 deletions(-) delete mode 100644 src/py1/mod.rs diff --git a/src/py/api.rs b/src/py/api.rs index d13ece9e..43b4bc60 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -6,7 +6,7 @@ use pyo3::wrap_pyfunction; use pyo3::PyObject; use super::py_int_allocator::PyIntAllocator; -use super::py_na_node::PyNaNode; +use super::py_na_node::PyNode; use super::py_native_mapping::{native_for_py, new_mapping, py_for_native}; use super::run_program::{__pyo3_get_function_deserialize_and_run_program, STRICT_MODE}; @@ -32,7 +32,7 @@ fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult(py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyCell> { +fn serialize_from_bytes<'p>(py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyCell> { let py_int_allocator = PyCell::new(py, PyIntAllocator::default())?; let allocator: &mut IntAllocator = &mut py_int_allocator.borrow_mut().arena; let cache = new_mapping(py)?; @@ -43,7 +43,7 @@ fn serialize_from_bytes<'p>(py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyCell< use crate::node::Node; #[pyfunction] -fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { +fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { let py_int_allocator_cell = PyCell::new(py, PyIntAllocator::default())?; let py_int_allocator: &mut PyIntAllocator = &mut py_int_allocator_cell.borrow_mut(); let allocator: &mut IntAllocator = &mut py_int_allocator.arena; @@ -74,7 +74,7 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; //m.add_function(wrap_pyfunction!(serialized_length, m)?)?; - m.add_class::()?; + m.add_class::()?; //m.add_function(wrap_pyfunction!(raise_eval_error, m)?)?; @@ -88,8 +88,8 @@ use crate::reduction::{EvalErr, Reduction}; #[allow(clippy::too_many_arguments)] pub fn py_run_program( py: Python, - program: &PyCell, - args: &PyCell, + program: &PyCell, + args: &PyCell, quote_kw: u8, apply_kw: u8, max_cost: Cost, diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index d457e58c..c75616a9 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -17,7 +17,7 @@ use crate::reduction::{EvalErr, Reduction, Response}; use crate::run_program::OperatorHandler; use super::f_table::{f_lookup_for_hashmap, FLookup}; -use super::py_na_node::PyNaNode; +use super::py_na_node::PyNode; use super::py_native_mapping::{native_for_py, py_for_native}; pub struct PyOperatorHandler { @@ -72,7 +72,7 @@ impl PyOperatorHandler { let i0: u32 = unwrap_or_eval_err(pair.get_item(0).extract(), args, "expected u32")?; - let py_node: &PyCell = + let py_node: &PyCell = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; let r = native_for_py(py, &self.cache, py_node, allocator); @@ -113,7 +113,7 @@ fn eval_err_for_pyerr<'p>( ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; - let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; + let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; let node: i32 = native_for_py(py, &cache, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) diff --git a/src/py/py_na_node.rs b/src/py/py_na_node.rs index 13e0569e..bb40aeb9 100644 --- a/src/py/py_na_node.rs +++ b/src/py/py_na_node.rs @@ -4,18 +4,18 @@ use pyo3::types::{PyBytes, PyTuple, PyType}; use super::py_view::PyView; #[pyclass(weakref, subclass)] -pub struct PyNaNode { +pub struct PyNode { pub py_view: Option, } -impl PyNaNode { +impl PyNode { pub fn new(py: Python, py_view: Option) -> PyResult<&PyCell> { - PyCell::new(py, PyNaNode { py_view }) + PyCell::new(py, PyNode { py_view }) } } #[pymethods] -impl PyNaNode { +impl PyNode { #[new] fn new_obj(py: Python, obj: &PyAny) -> PyResult { Ok(if let Ok(tuple) = obj.extract() { @@ -42,8 +42,8 @@ impl PyNaNode { fn new_pair<'p>( _cls: &PyType, py: Python<'p>, - p1: &PyCell, - p2: &PyCell, + p1: &PyCell, + p2: &PyCell, ) -> PyResult<&'p PyCell> { let tuple = PyTuple::new(py, &[p1, p2]); let py_view = PyView::new_pair(py, tuple)?; diff --git a/src/py/py_native_mapping.rs b/src/py/py_native_mapping.rs index d070850c..855175ec 100644 --- a/src/py/py_native_mapping.rs +++ b/src/py/py_native_mapping.rs @@ -6,7 +6,7 @@ use crate::int_allocator::IntAllocator; use super::py_view::PyView; -use super::py_na_node::PyNaNode; +use super::py_na_node::PyNode; pub fn new_mapping(py: Python) -> PyResult { Ok(py.eval("dict()", None, None)?.to_object(py)) @@ -15,7 +15,7 @@ pub fn new_mapping(py: Python) -> PyResult { pub fn add( py: Python, cache: &PyObject, - obj: &PyCell, + obj: &PyCell, ptr: &::Ptr, ) -> PyResult<()> { let locals = [ @@ -33,7 +33,7 @@ pub fn add( fn from_py_to_native_cache<'p>( py: Python<'p>, cache: &PyObject, - obj: &PyCell, + obj: &PyCell, ) -> PyResult<::Ptr> { let locals = [("cache", cache.clone()), ("key", obj.to_object(py))].into_py_dict(py); py.eval("cache.get(key)", None, Some(locals))?.extract() @@ -42,11 +42,11 @@ fn from_py_to_native_cache<'p>( fn populate_native( py: Python, cache: &PyObject, - obj: &PyCell, + obj: &PyCell, allocator: &mut IntAllocator, ) -> PyResult<::Ptr> { apply_to_tree(obj.to_object(py), move |obj| { - let node: &PyCell = obj.extract(py)?; + let node: &PyCell = obj.extract(py)?; // is it in cache yet? if from_py_to_native_cache(py, cache, node).is_ok() { @@ -67,8 +67,8 @@ fn populate_native( Some(PyView::Pair(pair)) => { let pair: &PyAny = pair.clone().into_ref(py); let pair: &PyTuple = pair.extract()?; - let p0: &PyCell = pair.get_item(0).extract()?; - let p1: &PyCell = pair.get_item(1).extract()?; + let p0: &PyCell = pair.get_item(0).extract()?; + let p1: &PyCell = pair.get_item(1).extract()?; let ptr_0: PyResult = from_py_to_native_cache(py, cache, p0); let ptr_1: PyResult = from_py_to_native_cache(py, cache, p1); if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { @@ -89,7 +89,7 @@ fn populate_native( pub fn native_for_py( py: Python, cache: &PyObject, - obj: &PyCell, + obj: &PyCell, allocator: &mut IntAllocator, ) -> PyResult<::Ptr> { from_py_to_native_cache(py, cache, obj) @@ -102,7 +102,7 @@ fn from_native_to_py_cache<'p>( py: Python<'p>, cache: &PyObject, ptr: &::Ptr, -) -> PyResult<&'p PyCell> { +) -> PyResult<&'p PyCell> { let locals = [("cache", cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); py.eval("cache[key]", None, Some(locals))?.extract() } @@ -112,7 +112,7 @@ fn populate_python<'p>( cache: &PyObject, ptr: &::Ptr, allocator: &mut IntAllocator, -) -> PyResult<&'p PyCell> { +) -> PyResult<&'p PyCell> { apply_to_tree(*ptr, move |ptr| { // is it in cache yet? if from_native_to_py_cache(py, cache, &ptr).is_ok() { @@ -130,7 +130,7 @@ fn populate_python<'p>( add( py, cache, - PyNaNode::new(py, Some(PyView::new_atom(py, py_bytes)))?, + PyNode::new(py, Some(PyView::new_atom(py, py_bytes)))?, &ptr, )?; Ok(None) @@ -153,11 +153,11 @@ fn populate_python<'p>( // the children are in the cache, create new node & populate cache with it Ok(tuple) => { - let (p1, p2): (&PyCell, &PyCell) = tuple.extract()?; + let (p1, p2): (&PyCell, &PyCell) = tuple.extract()?; add( py, cache, - PyNaNode::new( + PyNode::new( py, Some(PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?), )?, @@ -178,7 +178,7 @@ pub fn py_for_native<'p>( cache: &PyObject, ptr: &::Ptr, allocator: &mut IntAllocator, -) -> PyResult<&'p PyCell> { +) -> PyResult<&'p PyCell> { from_native_to_py_cache(py, cache, ptr) .or_else(|_err| Ok(populate_python(py, cache, ptr, allocator)?)) } diff --git a/src/py/py_view.rs b/src/py/py_view.rs index 16c78143..aacfd825 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -2,7 +2,7 @@ use pyo3::pycell::PyCell; use pyo3::types::{PyBytes, PyTuple}; use pyo3::{PyObject, PyResult, Python, ToPyObject}; -use super::py_na_node::PyNaNode; +use super::py_na_node::PyNode; #[derive(Clone)] pub enum PyView { @@ -19,8 +19,8 @@ impl PyView { if pair.len() != 2 { py.eval("raise ValueError('new_pair requires 2-tuple')", None, None)?; } - let _p0: &PyCell = pair.get_item(0).extract()?; - let _p1: &PyCell = pair.get_item(1).extract()?; + let _p0: &PyCell = pair.get_item(0).extract()?; + let _p1: &PyCell = pair.get_item(1).extract()?; Ok(PyView::Pair(pair.to_object(py))) } diff --git a/src/py1/mod.rs b/src/py1/mod.rs deleted file mode 100644 index d4dca94b..00000000 --- a/src/py1/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub mod api; -pub mod arc_allocator; -pub mod f_table; -pub mod glue; -pub mod native_op_lookup; -pub mod py_node; -pub mod run_program; -pub mod to_py_node; From 7947f4ef64ac660b5c141d9fa9d193fc6123135b Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 17 Mar 2021 17:27:36 -0700 Subject: [PATCH 32/76] More simplification. --- src/py/api.rs | 47 ++++---- src/py/mod.rs | 1 - src/py/op_fn.rs | 26 ++--- src/py/py_int_allocator.rs | 225 +++++++++++++++++++++++++++++++++++- src/py/py_native_mapping.rs | 212 --------------------------------- src/py/run_program.rs | 36 +++--- src/run_program.rs | 16 +-- 7 files changed, 274 insertions(+), 289 deletions(-) delete mode 100644 src/py/py_native_mapping.rs diff --git a/src/py/api.rs b/src/py/api.rs index 43b4bc60..34f63eb8 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -1,3 +1,4 @@ +use std::cell::RefMut; use std::collections::HashMap; use pyo3::prelude::*; @@ -7,7 +8,6 @@ use pyo3::PyObject; use super::py_int_allocator::PyIntAllocator; use super::py_na_node::PyNode; -use super::py_native_mapping::{native_for_py, new_mapping, py_for_native}; use super::run_program::{__pyo3_get_function_deserialize_and_run_program, STRICT_MODE}; use crate::int_allocator::IntAllocator; @@ -33,24 +33,22 @@ fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult(py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyCell> { - let py_int_allocator = PyCell::new(py, PyIntAllocator::default())?; - let allocator: &mut IntAllocator = &mut py_int_allocator.borrow_mut().arena; - let cache = new_mapping(py)?; + let py_int_allocator = PyIntAllocator::new(py)?.borrow(); + let mut allocator_refcell: RefMut = py_int_allocator.allocator(); + let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; let ptr = node_from_bytes(allocator, blob)?; - py_for_native(py, &cache, &ptr, allocator) + py_int_allocator.py_for_native(py, &ptr, allocator) } use crate::node::Node; #[pyfunction] fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { - let py_int_allocator_cell = PyCell::new(py, PyIntAllocator::default())?; - let py_int_allocator: &mut PyIntAllocator = &mut py_int_allocator_cell.borrow_mut(); - let allocator: &mut IntAllocator = &mut py_int_allocator.arena; + let py_int_allocator = PyIntAllocator::new(py)?.borrow(); + let mut allocator_refcell: RefMut = py_int_allocator.allocator(); + let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let mapping = new_mapping(py)?; - - let ptr = native_for_py(py, &mapping, sexp, allocator)?; + let ptr = py_int_allocator.native_for_py(py, sexp, allocator)?; let node = Node::new(allocator, ptr); let s: Vec = node_to_bytes(&node)?; @@ -86,8 +84,8 @@ use crate::reduction::{EvalErr, Reduction}; #[pyfunction] #[allow(clippy::too_many_arguments)] -pub fn py_run_program( - py: Python, +pub fn py_run_program<'p>( + py: Python<'p>, program: &PyCell, args: &PyCell, quote_kw: u8, @@ -96,28 +94,27 @@ pub fn py_run_program( opcode_lookup_by_name: HashMap>, py_callback: PyObject, ) -> PyResult<(Cost, PyObject)> { - let allocator: &mut IntAllocator = &mut IntAllocator::new(); + let py_int_allocator = PyIntAllocator::new(py)?.borrow(); + let mut allocator_refcell: RefMut = py_int_allocator.allocator(); + let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let cache = new_mapping(py)?; - let op_lookup = Box::new(PyOperatorHandler::new( - opcode_lookup_by_name, - py_callback, - cache.clone(), - )?); - let program = native_for_py(py, &cache, program, allocator)?; - let args = native_for_py(py, &cache, args, allocator)?; + let op_lookup = PyOperatorHandler::new(opcode_lookup_by_name, py_callback, &py_int_allocator)?; + let program = py_int_allocator.native_for_py(py, program, allocator)?; + let args = py_int_allocator.native_for_py(py, args, allocator)?; let r: Result, EvalErr> = crate::run_program::run_program( - allocator, &program, &args, quote_kw, apply_kw, max_cost, op_lookup, None, + allocator, &program, &args, quote_kw, apply_kw, max_cost, &op_lookup, None, ); match r { Ok(reduction) => { - let r = py_for_native(py, &cache, &reduction.1, allocator)?; + let r = py_int_allocator.py_for_native(py, &reduction.1, allocator)?; Ok((reduction.0, r.to_object(py))) } Err(eval_err) => { - let node: PyObject = py_for_native(py, &cache, &eval_err.0, allocator)?.to_object(py); + let node: PyObject = py_int_allocator + .py_for_native(py, &eval_err.0, allocator)? + .to_object(py); let s: String = eval_err.1; let s1: &str = &s; let msg: &PyString = PyString::new(py, s1); diff --git a/src/py/mod.rs b/src/py/mod.rs index 2eabf6f6..6b425009 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -3,6 +3,5 @@ pub mod f_table; pub mod op_fn; pub mod py_int_allocator; pub mod py_na_node; -pub mod py_native_mapping; pub mod py_view; pub mod run_program; diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index c75616a9..e3e7d1d3 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -17,31 +17,29 @@ use crate::reduction::{EvalErr, Reduction, Response}; use crate::run_program::OperatorHandler; use super::f_table::{f_lookup_for_hashmap, FLookup}; +use super::py_int_allocator::PyIntAllocator; use super::py_na_node::PyNode; -use super::py_native_mapping::{native_for_py, py_for_native}; -pub struct PyOperatorHandler { +pub struct PyOperatorHandler<'p> { native_lookup: FLookup, py_callable: PyObject, - pub cache: PyObject, + py_int_allocator: &'p PyIntAllocator, } -impl PyOperatorHandler { +impl<'p> PyOperatorHandler<'p> { pub fn new( opcode_lookup_by_name: HashMap>, py_callable: PyObject, - cache: PyObject, + py_int_allocator: &'p PyIntAllocator, ) -> PyResult { let native_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); Ok(PyOperatorHandler { native_lookup, py_callable, - cache, + py_int_allocator, }) } -} -impl PyOperatorHandler { pub fn invoke_py_obj( &self, obj: PyObject, @@ -53,7 +51,7 @@ impl PyOperatorHandler { Python::with_gil(|py| { let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); let r = unwrap_or_eval_err( - py_for_native(py, &self.cache, args, allocator), + self.py_int_allocator.py_for_native(py, args, allocator), args, "can't uncache", )?; @@ -61,7 +59,7 @@ impl PyOperatorHandler { match r1 { Err(pyerr) => { let eval_err: PyResult> = - eval_err_for_pyerr(py, &pyerr, self.cache.clone(), allocator); + eval_err_for_pyerr(py, &pyerr, self.py_int_allocator, allocator); let r: EvalErr = unwrap_or_eval_err(eval_err, args, "unexpected exception")?; Err(r) @@ -75,7 +73,7 @@ impl PyOperatorHandler { let py_node: &PyCell = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = native_for_py(py, &self.cache, py_node, allocator); + let r = self.py_int_allocator.native_for_py(py, py_node, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } @@ -84,7 +82,7 @@ impl PyOperatorHandler { } } -impl OperatorHandler for PyOperatorHandler { +impl OperatorHandler for PyOperatorHandler<'_> { fn op( &self, allocator: &mut IntAllocator, @@ -108,13 +106,13 @@ impl OperatorHandler for PyOperatorHandler { fn eval_err_for_pyerr<'p>( py: Python<'p>, pyerr: &PyErr, - cache: PyObject, + py_int_allocator: &'p PyIntAllocator, allocator: &mut IntAllocator, ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = native_for_py(py, &cache, sexp, allocator)?; + let node: i32 = py_int_allocator.native_for_py(py, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) } diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs index 0b5c53f3..be0bc0e8 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_int_allocator.rs @@ -1,16 +1,231 @@ +use std::cell::{RefCell, RefMut}; + use pyo3::prelude::pyclass; +use pyo3::prelude::*; +use pyo3::types::{IntoPyDict, PyBytes, PyTuple}; +use crate::allocator::{Allocator, SExp}; use crate::int_allocator::IntAllocator; +use super::py_na_node::PyNode; +use super::py_view::PyView; + #[pyclass(subclass, unsendable)] pub struct PyIntAllocator { - pub arena: IntAllocator, + arena: RefCell, + cache: PyObject, +} + +impl PyIntAllocator { + pub fn new<'p>(py: Python<'p>) -> PyResult<&'p PyCell> { + Ok(PyCell::new( + py, + PyIntAllocator { + arena: RefCell::new(IntAllocator::default()), + cache: py.eval("dict()", None, None)?.to_object(py), + }, + )?) + } + + pub fn allocator(&self) -> RefMut { + self.arena.borrow_mut() + } + + pub fn add( + &self, + py: Python, + obj: &PyCell, + ptr: &::Ptr, + ) -> PyResult<()> { + let locals = [ + ("cache", self.cache.clone()), + ("obj", obj.to_object(py)), + ("ptr", ptr.to_object(py)), + ] + .into_py_dict(py); + + py.run("cache[ptr] = obj; cache[obj] = ptr", None, Some(locals)) + } + + // py to native methods + + fn from_py_to_native_cache<'p>( + &self, + py: Python<'p>, + obj: &PyCell, + ) -> PyResult<::Ptr> { + let locals = [("cache", self.cache.clone()), ("key", obj.to_object(py))].into_py_dict(py); + py.eval("cache.get(key)", None, Some(locals))?.extract() + } + + fn populate_native( + &self, + py: Python, + obj: &PyCell, + allocator: &mut IntAllocator, + ) -> PyResult<::Ptr> { + apply_to_tree(obj.to_object(py), move |obj| { + let node: &PyCell = obj.extract(py)?; + + // is it in cache yet? + if self.from_py_to_native_cache(py, node).is_ok() { + // yep, we're done + return Ok(None); + } + + // it's not in the cache + + match &node.borrow().py_view { + Some(PyView::Atom(obj)) => { + let blob: &[u8] = obj.extract(py).unwrap(); + let ptr = allocator.new_atom(blob).unwrap(); + self.add(py, node, &ptr)?; + + Ok(None) + } + Some(PyView::Pair(pair)) => { + let pair: &PyAny = pair.clone().into_ref(py); + let pair: &PyTuple = pair.extract()?; + let p0: &PyCell = pair.get_item(0).extract()?; + let p1: &PyCell = pair.get_item(1).extract()?; + let ptr_0: PyResult = self.from_py_to_native_cache(py, p0); + let ptr_1: PyResult = self.from_py_to_native_cache(py, p1); + if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { + let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); + self.add(py, node, &ptr)?; + Ok(None) + } else { + Ok(Some((p0.to_object(py), p1.to_object(py)))) + } + } + _ => py_raise(py, "py view is None"), + } + })?; + + self.from_py_to_native_cache(py, obj) + } + + pub fn native_for_py( + &self, + py: Python, + obj: &PyCell, + allocator: &mut IntAllocator, + ) -> PyResult<::Ptr> { + self.from_py_to_native_cache(py, obj) + .or_else(|_err| self.populate_native(py, obj, allocator)) + } + + // native to py methods + + fn from_native_to_py_cache<'p>( + &self, + py: Python<'p>, + ptr: &::Ptr, + ) -> PyResult<&'p PyCell> { + let locals = [("cache", self.cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); + py.eval("cache[key]", None, Some(locals))?.extract() + } + + fn populate_python<'p>( + &self, + py: Python<'p>, + ptr: &::Ptr, + allocator: &mut IntAllocator, + ) -> PyResult<&'p PyCell> { + apply_to_tree(*ptr, move |ptr| { + // is it in cache yet? + if self.from_native_to_py_cache(py, &ptr).is_ok() { + // yep, we're done + return Ok(None); + } + + // it's not in the cache + + match allocator.sexp(&ptr) { + SExp::Atom(a) => { + // it's an atom, so we just populate cache directly + let blob = allocator.buf(&a); + let py_bytes = PyBytes::new(py, blob); + self.add( + py, + PyNode::new(py, Some(PyView::new_atom(py, py_bytes)))?, + &ptr, + )?; + Ok(None) + } + SExp::Pair(ptr_1, ptr_2) => { + // we can only create this if the children are in the cache + // Let's fine out + let locals = [ + ("cache", self.cache.clone()), + ("p1", ptr_1.to_object(py)), + ("p2", ptr_2.to_object(py)), + ] + .into_py_dict(py); + + let pair: PyResult<&PyAny> = + py.eval("(cache[p1], cache[p2])", None, Some(locals)); + + match pair { + // the children aren't in the cache, keep drilling down + Err(_) => Ok(Some((ptr_1, ptr_2))), + + // the children are in the cache, create new node & populate cache with it + Ok(tuple) => { + let (p1, p2): (&PyCell, &PyCell) = tuple.extract()?; + self.add( + py, + PyNode::new( + py, + Some(PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?), + )?, + &ptr, + )?; + Ok(None) + } + } + } + } + })?; + + self.from_native_to_py_cache(py, &ptr) + } + + pub fn py_for_native<'p>( + &self, + py: Python<'p>, + ptr: &::Ptr, + allocator: &mut IntAllocator, + ) -> PyResult<&'p PyCell> { + self.from_native_to_py_cache(py, ptr) + .or_else(|_err| Ok(self.populate_python(py, ptr, allocator)?)) + } } -impl Default for PyIntAllocator { - fn default() -> Self { - PyIntAllocator { - arena: IntAllocator::default(), +fn apply_to_tree(node: T, mut apply: F) -> PyResult<()> +where + F: FnMut(T) -> PyResult>, + T: Clone, +{ + let mut items = vec![node]; + loop { + let t = items.pop(); + if let Some(obj) = t { + if let Some((p0, p1)) = apply(obj.clone())? { + items.push(obj); + items.push(p0); + items.push(p1); + } + } else { + break; } } + Ok(()) +} + +fn py_raise(py: Python, msg: &str) -> PyResult { + let locals = [("msg", msg.to_object(py))].into_py_dict(py); + + py.run("raise RuntimeError(msg)", None, Some(locals))?; + panic!("we should never get here") } diff --git a/src/py/py_native_mapping.rs b/src/py/py_native_mapping.rs deleted file mode 100644 index 855175ec..00000000 --- a/src/py/py_native_mapping.rs +++ /dev/null @@ -1,212 +0,0 @@ -use pyo3::prelude::*; -use pyo3::types::{IntoPyDict, PyBytes, PyTuple}; - -use crate::allocator::{Allocator, SExp}; -use crate::int_allocator::IntAllocator; - -use super::py_view::PyView; - -use super::py_na_node::PyNode; - -pub fn new_mapping(py: Python) -> PyResult { - Ok(py.eval("dict()", None, None)?.to_object(py)) -} - -pub fn add( - py: Python, - cache: &PyObject, - obj: &PyCell, - ptr: &::Ptr, -) -> PyResult<()> { - let locals = [ - ("cache", cache.clone()), - ("obj", obj.to_object(py)), - ("ptr", ptr.to_object(py)), - ] - .into_py_dict(py); - - py.run("cache[ptr] = obj; cache[obj] = ptr", None, Some(locals)) -} - -// py to native methods - -fn from_py_to_native_cache<'p>( - py: Python<'p>, - cache: &PyObject, - obj: &PyCell, -) -> PyResult<::Ptr> { - let locals = [("cache", cache.clone()), ("key", obj.to_object(py))].into_py_dict(py); - py.eval("cache.get(key)", None, Some(locals))?.extract() -} - -fn populate_native( - py: Python, - cache: &PyObject, - obj: &PyCell, - allocator: &mut IntAllocator, -) -> PyResult<::Ptr> { - apply_to_tree(obj.to_object(py), move |obj| { - let node: &PyCell = obj.extract(py)?; - - // is it in cache yet? - if from_py_to_native_cache(py, cache, node).is_ok() { - // yep, we're done - return Ok(None); - } - - // it's not in the cache - - match &node.borrow().py_view { - Some(PyView::Atom(obj)) => { - let blob: &[u8] = obj.extract(py).unwrap(); - let ptr = allocator.new_atom(blob).unwrap(); - add(py, cache, node, &ptr)?; - - Ok(None) - } - Some(PyView::Pair(pair)) => { - let pair: &PyAny = pair.clone().into_ref(py); - let pair: &PyTuple = pair.extract()?; - let p0: &PyCell = pair.get_item(0).extract()?; - let p1: &PyCell = pair.get_item(1).extract()?; - let ptr_0: PyResult = from_py_to_native_cache(py, cache, p0); - let ptr_1: PyResult = from_py_to_native_cache(py, cache, p1); - if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { - let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); - add(py, cache, node, &ptr)?; - Ok(None) - } else { - Ok(Some((p0.to_object(py), p1.to_object(py)))) - } - } - _ => py_raise(py, "py view is None"), - } - })?; - - from_py_to_native_cache(py, cache, obj) -} - -pub fn native_for_py( - py: Python, - cache: &PyObject, - obj: &PyCell, - allocator: &mut IntAllocator, -) -> PyResult<::Ptr> { - from_py_to_native_cache(py, cache, obj) - .or_else(|_err| populate_native(py, cache, obj, allocator)) -} - -// native to py methods - -fn from_native_to_py_cache<'p>( - py: Python<'p>, - cache: &PyObject, - ptr: &::Ptr, -) -> PyResult<&'p PyCell> { - let locals = [("cache", cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); - py.eval("cache[key]", None, Some(locals))?.extract() -} - -fn populate_python<'p>( - py: Python<'p>, - cache: &PyObject, - ptr: &::Ptr, - allocator: &mut IntAllocator, -) -> PyResult<&'p PyCell> { - apply_to_tree(*ptr, move |ptr| { - // is it in cache yet? - if from_native_to_py_cache(py, cache, &ptr).is_ok() { - // yep, we're done - return Ok(None); - } - - // it's not in the cache - - match allocator.sexp(&ptr) { - SExp::Atom(a) => { - // it's an atom, so we just populate cache directly - let blob = allocator.buf(&a); - let py_bytes = PyBytes::new(py, blob); - add( - py, - cache, - PyNode::new(py, Some(PyView::new_atom(py, py_bytes)))?, - &ptr, - )?; - Ok(None) - } - SExp::Pair(ptr_1, ptr_2) => { - // we can only create this if the children are in the cache - // Let's fine out - let locals = [ - ("cache", cache.clone()), - ("p1", ptr_1.to_object(py)), - ("p2", ptr_2.to_object(py)), - ] - .into_py_dict(py); - - let pair: PyResult<&PyAny> = py.eval("(cache[p1], cache[p2])", None, Some(locals)); - - match pair { - // the children aren't in the cache, keep drilling down - Err(_) => Ok(Some((ptr_1, ptr_2))), - - // the children are in the cache, create new node & populate cache with it - Ok(tuple) => { - let (p1, p2): (&PyCell, &PyCell) = tuple.extract()?; - add( - py, - cache, - PyNode::new( - py, - Some(PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?), - )?, - &ptr, - )?; - Ok(None) - } - } - } - } - })?; - - from_native_to_py_cache(py, cache, &ptr) -} - -pub fn py_for_native<'p>( - py: Python<'p>, - cache: &PyObject, - ptr: &::Ptr, - allocator: &mut IntAllocator, -) -> PyResult<&'p PyCell> { - from_native_to_py_cache(py, cache, ptr) - .or_else(|_err| Ok(populate_python(py, cache, ptr, allocator)?)) -} - -fn apply_to_tree(node: T, mut apply: F) -> PyResult<()> -where - F: FnMut(T) -> PyResult>, - T: Clone, -{ - let mut items = vec![node]; - loop { - let t = items.pop(); - if let Some(obj) = t { - if let Some((p0, p1)) = apply(obj.clone())? { - items.push(obj); - items.push(p0); - items.push(p1); - } - } else { - break; - } - } - Ok(()) -} - -fn py_raise(py: Python, msg: &str) -> PyResult { - let locals = [("msg", msg.to_object(py))].into_py_dict(py); - - py.run("raise RuntimeError(msg)", None, Some(locals))?; - panic!("we should never get here") -} diff --git a/src/py/run_program.rs b/src/py/run_program.rs index 271e358d..b74c4cfb 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -1,3 +1,4 @@ +use std::cell::RefMut; use std::collections::HashMap; use crate::allocator::Allocator; @@ -11,7 +12,7 @@ use crate::run_program::{run_program, OperatorHandler}; use crate::serialize::{node_from_bytes, node_to_bytes, serialized_length_from_bytes}; use super::f_table::{f_lookup_for_hashmap, FLookup}; -use super::py_native_mapping::{new_mapping, py_for_native}; +use super::py_int_allocator::PyIntAllocator; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict}; @@ -59,34 +60,27 @@ pub fn deserialize_and_run_program( max_cost: Cost, flags: u32, ) -> PyResult<(Cost, PyObject)> { - let mut allocator = IntAllocator::new(); + let py_int_allocator = PyIntAllocator::new(py)?.borrow(); + let mut allocator_refcell: RefMut = py_int_allocator.allocator(); + let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; let f_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); let strict: bool = (flags & STRICT_MODE) != 0; - let f: Box + Send> = - Box::new(OperatorHandlerWithMode { f_lookup, strict }); - let program = node_from_bytes(&mut allocator, program)?; - let args = node_from_bytes(&mut allocator, args)?; - let cache = new_mapping(py)?; + let f = OperatorHandlerWithMode { f_lookup, strict }; + let program = node_from_bytes(allocator, program)?; + let args = node_from_bytes(allocator, args)?; - let r = py.allow_threads(|| { - run_program( - &mut allocator, - &program, - &args, - quote_kw, - apply_kw, - max_cost, - f, - None, - ) - }); + let r = run_program( + allocator, &program, &args, quote_kw, apply_kw, max_cost, &f, None, + ); match r { Ok(reduction) => Ok(( reduction.0, - py_for_native(py, &cache, &reduction.1, &mut allocator)?.to_object(py), + py_int_allocator + .py_for_native(py, &reduction.1, allocator)? + .to_object(py), )), Err(eval_err) => { - let node_as_blob = node_to_bytes(&Node::new(&allocator, eval_err.0))?; + let node_as_blob = node_to_bytes(&Node::new(allocator, eval_err.0))?; let msg = eval_err.1; let ctx: &PyDict = PyDict::new(py); ctx.set_item("msg", msg)?; diff --git a/src/run_program.rs b/src/run_program.rs index 90720666..4e6721d7 100644 --- a/src/run_program.rs +++ b/src/run_program.rs @@ -53,7 +53,7 @@ pub struct RunProgramContext<'a, T: Allocator> { allocator: &'a mut T, quote_kw: u8, apply_kw: u8, - operator_lookup: Box>, + operator_lookup: &'a dyn OperatorHandler, pre_eval: Option>, posteval_stack: Vec>>, val_stack: Vec, @@ -161,7 +161,7 @@ impl<'a, 'h, T: Allocator> RunProgramContext<'a, T> { allocator: &'a mut T, quote_kw: u8, apply_kw: u8, - operator_lookup: Box>, + operator_lookup: &'a dyn OperatorHandler, pre_eval: Option>, ) -> Self { RunProgramContext { @@ -195,10 +195,7 @@ impl<'a, 'h, T: Allocator> RunProgramContext<'a, T> { } } -impl<'a, T: Allocator> RunProgramContext<'a, T> -where - ::Ptr: 'static, -{ +impl<'a, T: Allocator> RunProgramContext<'a, T> { fn eval_op_atom( &mut self, op_buf: &T::AtomBuf, @@ -380,12 +377,9 @@ pub fn run_program( quote_kw: u8, apply_kw: u8, max_cost: Cost, - operator_lookup: Box>, + operator_lookup: &dyn OperatorHandler, pre_eval: Option>, -) -> Response -where - ::Ptr: 'static, -{ +) -> Response { let mut rpc = RunProgramContext::new(allocator, quote_kw, apply_kw, operator_lookup, pre_eval); rpc.run_program(program, args, max_cost) } From f794a2664d9862aa156ac9b5ed21b6c4ba59763f Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 17 Mar 2021 17:31:14 -0700 Subject: [PATCH 33/76] Hash by id. --- src/py/py_int_allocator.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs index be0bc0e8..03035c05 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_int_allocator.rs @@ -17,7 +17,7 @@ pub struct PyIntAllocator { } impl PyIntAllocator { - pub fn new<'p>(py: Python<'p>) -> PyResult<&'p PyCell> { + pub fn new(py: Python) -> PyResult<&PyCell> { Ok(PyCell::new( py, PyIntAllocator { @@ -44,7 +44,7 @@ impl PyIntAllocator { ] .into_py_dict(py); - py.run("cache[ptr] = obj; cache[obj] = ptr", None, Some(locals)) + py.run("cache[ptr] = obj; cache[id(obj)] = ptr", None, Some(locals)) } // py to native methods @@ -55,7 +55,7 @@ impl PyIntAllocator { obj: &PyCell, ) -> PyResult<::Ptr> { let locals = [("cache", self.cache.clone()), ("key", obj.to_object(py))].into_py_dict(py); - py.eval("cache.get(key)", None, Some(locals))?.extract() + py.eval("cache.get(id(key))", None, Some(locals))?.extract() } fn populate_native( From 3dd28656e7fa4276c8f58d074c3d442d8648d647 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 17 Mar 2021 18:29:14 -0700 Subject: [PATCH 34/76] Rename file. --- src/py/api.rs | 2 +- src/py/mod.rs | 2 +- src/py/op_fn.rs | 2 +- src/py/py_int_allocator.rs | 2 +- src/py/{py_na_node.rs => py_node.rs} | 0 src/py/py_view.rs | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename src/py/{py_na_node.rs => py_node.rs} (100%) diff --git a/src/py/api.rs b/src/py/api.rs index 34f63eb8..7b6bc15b 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -7,7 +7,7 @@ use pyo3::wrap_pyfunction; use pyo3::PyObject; use super::py_int_allocator::PyIntAllocator; -use super::py_na_node::PyNode; +use super::py_node::PyNode; use super::run_program::{__pyo3_get_function_deserialize_and_run_program, STRICT_MODE}; use crate::int_allocator::IntAllocator; diff --git a/src/py/mod.rs b/src/py/mod.rs index 6b425009..c344968a 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -2,6 +2,6 @@ pub mod api; pub mod f_table; pub mod op_fn; pub mod py_int_allocator; -pub mod py_na_node; +pub mod py_node; pub mod py_view; pub mod run_program; diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index e3e7d1d3..1e6c86cf 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -18,7 +18,7 @@ use crate::run_program::OperatorHandler; use super::f_table::{f_lookup_for_hashmap, FLookup}; use super::py_int_allocator::PyIntAllocator; -use super::py_na_node::PyNode; +use super::py_node::PyNode; pub struct PyOperatorHandler<'p> { native_lookup: FLookup, diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs index 03035c05..a053a433 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_int_allocator.rs @@ -7,7 +7,7 @@ use pyo3::types::{IntoPyDict, PyBytes, PyTuple}; use crate::allocator::{Allocator, SExp}; use crate::int_allocator::IntAllocator; -use super::py_na_node::PyNode; +use super::py_node::PyNode; use super::py_view::PyView; #[pyclass(subclass, unsendable)] diff --git a/src/py/py_na_node.rs b/src/py/py_node.rs similarity index 100% rename from src/py/py_na_node.rs rename to src/py/py_node.rs diff --git a/src/py/py_view.rs b/src/py/py_view.rs index aacfd825..596a2f81 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -2,7 +2,7 @@ use pyo3::pycell::PyCell; use pyo3::types::{PyBytes, PyTuple}; use pyo3::{PyObject, PyResult, Python, ToPyObject}; -use super::py_na_node::PyNode; +use super::py_node::PyNode; #[derive(Clone)] pub enum PyView { From 39e0853534b4889851515ee606a9554bf17d24d6 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 17 Mar 2021 18:36:59 -0700 Subject: [PATCH 35/76] Make `PyView` non-optional. --- src/py/py_int_allocator.rs | 16 ++++------------ src/py/py_node.rs | 31 +++++++++++++------------------ 2 files changed, 17 insertions(+), 30 deletions(-) diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs index a053a433..0c7a1ba2 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_int_allocator.rs @@ -76,14 +76,14 @@ impl PyIntAllocator { // it's not in the cache match &node.borrow().py_view { - Some(PyView::Atom(obj)) => { + PyView::Atom(obj) => { let blob: &[u8] = obj.extract(py).unwrap(); let ptr = allocator.new_atom(blob).unwrap(); self.add(py, node, &ptr)?; Ok(None) } - Some(PyView::Pair(pair)) => { + PyView::Pair(pair) => { let pair: &PyAny = pair.clone().into_ref(py); let pair: &PyTuple = pair.extract()?; let p0: &PyCell = pair.get_item(0).extract()?; @@ -98,7 +98,6 @@ impl PyIntAllocator { Ok(Some((p0.to_object(py), p1.to_object(py)))) } } - _ => py_raise(py, "py view is None"), } })?; @@ -148,7 +147,7 @@ impl PyIntAllocator { let py_bytes = PyBytes::new(py, blob); self.add( py, - PyNode::new(py, Some(PyView::new_atom(py, py_bytes)))?, + PyNode::new(py, PyView::new_atom(py, py_bytes))?, &ptr, )?; Ok(None) @@ -177,7 +176,7 @@ impl PyIntAllocator { py, PyNode::new( py, - Some(PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?), + PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?, )?, &ptr, )?; @@ -222,10 +221,3 @@ where } Ok(()) } - -fn py_raise(py: Python, msg: &str) -> PyResult { - let locals = [("msg", msg.to_object(py))].into_py_dict(py); - - py.run("raise RuntimeError(msg)", None, Some(locals))?; - panic!("we should never get here") -} diff --git a/src/py/py_node.rs b/src/py/py_node.rs index bb40aeb9..314c4635 100644 --- a/src/py/py_node.rs +++ b/src/py/py_node.rs @@ -5,11 +5,11 @@ use super::py_view::PyView; #[pyclass(weakref, subclass)] pub struct PyNode { - pub py_view: Option, + pub py_view: PyView, } impl PyNode { - pub fn new(py: Python, py_view: Option) -> PyResult<&PyCell> { + pub fn new(py: Python, py_view: PyView) -> PyResult<&PyCell> { PyCell::new(py, PyNode { py_view }) } } @@ -20,22 +20,18 @@ impl PyNode { fn new_obj(py: Python, obj: &PyAny) -> PyResult { Ok(if let Ok(tuple) = obj.extract() { let py_view = PyView::new_pair(py, tuple)?; - Self { - py_view: Some(py_view), - } + Self { py_view } } else { let py_bytes: &PyBytes = obj.extract()?; let py_view = PyView::new_atom(py, py_bytes); - Self { - py_view: Some(py_view), - } + Self { py_view } }) } #[classmethod] fn new_atom<'p>(_cls: &PyType, py: Python<'p>, atom: &PyBytes) -> PyResult<&'p PyCell> { let py_view = PyView::new_atom(py, atom); - Self::new(py, Some(py_view)) + Self::new(py, py_view) } #[classmethod] @@ -47,27 +43,27 @@ impl PyNode { ) -> PyResult<&'p PyCell> { let tuple = PyTuple::new(py, &[p1, p2]); let py_view = PyView::new_pair(py, tuple)?; - Self::new(py, Some(py_view)) + Self::new(py, py_view) } #[classmethod] fn new_tuple<'p>(_cls: &PyType, py: Python<'p>, tuple: &PyTuple) -> PyResult<&'p PyCell> { let py_view = PyView::new_pair(py, tuple)?; - Self::new(py, Some(py_view)) + Self::new(py, py_view) } #[getter(atom)] pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - match slf.try_borrow()?.py_view.as_ref() { - Some(PyView::Atom(obj)) => Ok(obj.clone()), + match &slf.try_borrow()?.py_view { + PyView::Atom(obj) => Ok(obj.clone()), _ => Ok(py.None()), } } #[getter(pair)] pub fn pair<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - match slf.try_borrow()?.py_view.as_ref() { - Some(PyView::Pair(obj)) => Ok(obj.clone()), + match &slf.try_borrow()?.py_view { + PyView::Pair(obj) => Ok(obj.clone()), _ => Ok(py.None()), } } @@ -75,9 +71,8 @@ impl PyNode { #[getter(python)] pub fn python<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { Ok(match &slf.borrow().py_view { - Some(PyView::Atom(atom)) => ("Atom", atom).to_object(py), - Some(PyView::Pair(pair)) => ("Pair", pair).to_object(py), - _ => py.None(), + PyView::Atom(atom) => ("Atom", atom).to_object(py), + PyView::Pair(pair) => ("Pair", pair).to_object(py), }) } } From 5c5ffa01de3bfbb65c1890fae4af21388bf989b6 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 24 Mar 2021 15:07:02 -0700 Subject: [PATCH 36/76] Add `PyIntNode`. --- src/py/api.rs | 29 ++++++++++++++++++------- src/py/mod.rs | 1 + src/py/py_int_allocator.rs | 6 +----- src/py/py_int_node.rs | 44 ++++++++++++++++++++++++++++++++++++++ src/py/py_view.rs | 1 - 5 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 src/py/py_int_node.rs diff --git a/src/py/api.rs b/src/py/api.rs index 7b6bc15b..9ce2feef 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -7,6 +7,7 @@ use pyo3::wrap_pyfunction; use pyo3::PyObject; use super::py_int_allocator::PyIntAllocator; +use super::py_int_node::PyIntNode; use super::py_node::PyNode; use super::run_program::{__pyo3_get_function_deserialize_and_run_program, STRICT_MODE}; @@ -32,12 +33,23 @@ fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult(py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyCell> { - let py_int_allocator = PyIntAllocator::new(py)?.borrow(); - let mut allocator_refcell: RefMut = py_int_allocator.allocator(); - let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let ptr = node_from_bytes(allocator, blob)?; - py_int_allocator.py_for_native(py, &ptr, allocator) +fn deserialize_from_bytes_for_allocator<'p>( + py: Python<'p>, + blob: &[u8], + py_int_allocator: &PyCell, +) -> PyResult { + let ptr = { + let py_int_allocator: PyRef = py_int_allocator.borrow(); + let allocator: &mut IntAllocator = &mut py_int_allocator.allocator() as &mut IntAllocator; + node_from_bytes(allocator, blob)? + }; + Ok(PyIntNode::new(py, py_int_allocator, ptr)) +} + +#[pyfunction] +fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult { + let py_int_allocator = PyIntAllocator::new(py)?; + deserialize_from_bytes_for_allocator(py, blob, &py_int_allocator) } use crate::node::Node; @@ -59,7 +71,8 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p #[pymodule] fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(py_run_program, m)?)?; - m.add_function(wrap_pyfunction!(serialize_from_bytes, m)?)?; + m.add_function(wrap_pyfunction!(deserialize_from_bytes, m)?)?; + m.add_function(wrap_pyfunction!(deserialize_from_bytes_for_allocator, m)?)?; m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; @@ -68,7 +81,7 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { //m.add_class::()?; // m.add_class::()?; - //m.add_class::()?; + m.add_class::()?; m.add_class::()?; //m.add_function(wrap_pyfunction!(serialized_length, m)?)?; diff --git a/src/py/mod.rs b/src/py/mod.rs index c344968a..71c829c6 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -2,6 +2,7 @@ pub mod api; pub mod f_table; pub mod op_fn; pub mod py_int_allocator; +pub mod py_int_node; pub mod py_node; pub mod py_view; pub mod run_program; diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs index 0c7a1ba2..e6c351b5 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_int_allocator.rs @@ -145,11 +145,7 @@ impl PyIntAllocator { // it's an atom, so we just populate cache directly let blob = allocator.buf(&a); let py_bytes = PyBytes::new(py, blob); - self.add( - py, - PyNode::new(py, PyView::new_atom(py, py_bytes))?, - &ptr, - )?; + self.add(py, PyNode::new(py, PyView::new_atom(py, py_bytes))?, &ptr)?; Ok(None) } SExp::Pair(ptr_1, ptr_2) => { diff --git a/src/py/py_int_node.rs b/src/py/py_int_node.rs new file mode 100644 index 00000000..ae3b872b --- /dev/null +++ b/src/py/py_int_node.rs @@ -0,0 +1,44 @@ +use std::cell::RefMut; + +use pyo3::prelude::{pyclass, pymethods}; +use pyo3::PyCell; +use pyo3::PyObject; +use pyo3::PyRef; +use pyo3::PyResult; +use pyo3::Python; +use pyo3::ToPyObject; + +use crate::int_allocator::IntAllocator; + +use super::py_int_allocator::PyIntAllocator; +use super::py_node::PyNode; + +#[pyclass(weakref, subclass)] +pub struct PyIntNode { + py_int_allocator: PyObject, + ptr: i32, +} + +impl PyIntNode { + pub fn new(py: Python, py_int_allocator: &PyCell, ptr: i32) -> Self { + let py_int_allocator = py_int_allocator.to_object(py); + Self { + py_int_allocator, + ptr, + } + } +} + +#[pymethods] +impl PyIntNode { + fn clvm_object<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { + let py_int_allocator: PyRef = self.py_int_allocator.extract(py)?; + let mut allocator: RefMut = py_int_allocator.allocator(); + py_int_allocator.py_for_native(py, &self.ptr, &mut allocator) + } + + #[getter(arena)] + fn arena<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { + self.py_int_allocator.extract(py) + } +} diff --git a/src/py/py_view.rs b/src/py/py_view.rs index 596a2f81..15319ac9 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -4,7 +4,6 @@ use pyo3::{PyObject, PyResult, Python, ToPyObject}; use super::py_node::PyNode; -#[derive(Clone)] pub enum PyView { Atom(PyObject), Pair(PyObject), From 23bbb9c531435df92703e0b46ec6a7fa7b043373 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 25 Mar 2021 16:33:34 -0700 Subject: [PATCH 37/76] Checkpoint of `Dialect`. --- src/py/api.rs | 3 +- src/py/dialect.rs | 241 ++++++++++++++++++++++++++++++++++++++++++++ src/py/f_table.rs | 2 +- src/py/mod.rs | 2 + src/py/native_op.rs | 18 ++++ 5 files changed, 264 insertions(+), 2 deletions(-) create mode 100644 src/py/dialect.rs create mode 100644 src/py/native_op.rs diff --git a/src/py/api.rs b/src/py/api.rs index 9ce2feef..3924c81c 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -6,6 +6,7 @@ use pyo3::types::{PyBytes, PyDict, PyString}; use pyo3::wrap_pyfunction; use pyo3::PyObject; +use super::dialect::Dialect; use super::py_int_allocator::PyIntAllocator; use super::py_int_node::PyIntNode; use super::py_node::PyNode; @@ -79,7 +80,7 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add("STRICT_MODE", STRICT_MODE)?; //m.add_class::()?; - // m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/py/dialect.rs b/src/py/dialect.rs new file mode 100644 index 00000000..0084a3dc --- /dev/null +++ b/src/py/dialect.rs @@ -0,0 +1,241 @@ +use std::cell::RefMut; +use std::collections::HashMap; + +use pyo3::prelude::{pyclass, pymethods}; + +use pyo3::types::{PyBytes, PyDict, PyString, PyTuple}; +use pyo3::{PyCell, PyErr, PyObject, PyRef, PyResult, Python, ToPyObject}; + +use crate::allocator::Allocator; +use crate::cost::Cost; +use crate::err_utils::err; +use crate::int_allocator::IntAllocator; +use crate::more_ops::op_unknown; +use crate::reduction::EvalErr; +use crate::reduction::Reduction; +use crate::reduction::Response; +use crate::run_program::OperatorHandler; + +use super::f_table::FLookup; +use super::f_table::OpFn; +use super::native_op::NativeOp; +use super::py_int_allocator::PyIntAllocator; +use super::py_node::PyNode; + +#[pyclass] +pub struct Dialect { + quote_kw: u8, + apply_kw: u8, + u8_lookup: FLookup, + python_u8_lookup: HashMap, PyObject>, + native_u8_lookup: HashMap, OpFn>, + strict: bool, +} + +#[pymethods] +impl Dialect { + #[new] + pub fn new( + py: Python, + quote_kw: u8, + apply_kw: u8, + op_table: HashMap, PyObject>, + strict: bool, + ) -> PyResult { + let mut u8_lookup = [None; 256]; + let mut python_u8_lookup = HashMap::new(); + let mut native_u8_lookup = HashMap::new(); + for (op, fn_obj) in op_table.iter() { + let r: PyResult> = fn_obj.extract(py); + if let Ok(native_op) = r { + if op.len() == 1 { + let index = op[0] as usize; + u8_lookup[index] = Some(native_op.op); + } else { + native_u8_lookup.insert(op.to_owned(), native_op.op); + } + } else { + python_u8_lookup.insert(op.to_owned(), fn_obj.clone()); + } + } + Ok(Self { + quote_kw, + apply_kw, + u8_lookup, + python_u8_lookup, + native_u8_lookup, + strict, + }) + } + + pub fn run_program<'p>( + &self, + py: Python<'p>, + program: &PyCell, + args: &PyCell, + max_cost: Cost, + ) -> PyResult<(Cost, PyObject)> { + let py_int_allocator = PyIntAllocator::new(py)?; + let py_int_allocator = &py_int_allocator.borrow() as &PyIntAllocator; + let drc = DialectRunningContext { + dialect: self, + py_int_allocator, + }; + + let mut allocator_refcell: RefMut = py_int_allocator.allocator(); + let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; + + let program = py_int_allocator.native_for_py(py, program, allocator)?; + let args = py_int_allocator.native_for_py(py, args, allocator)?; + + let r: Result, EvalErr> = crate::run_program::run_program( + allocator, + &program, + &args, + self.quote_kw, + self.apply_kw, + max_cost, + &drc, + None, + ); + + match r { + Ok(reduction) => { + let r = py_int_allocator.py_for_native(py, &reduction.1, allocator)?; + Ok((reduction.0, r.to_object(py))) + } + Err(eval_err) => { + let node: PyObject = py_int_allocator + .py_for_native(py, &eval_err.0, allocator)? + .to_object(py); + let s: String = eval_err.1; + let s1: &str = &s; + let msg: &PyString = PyString::new(py, s1); + match raise_eval_error(py, &msg, node) { + Err(x) => Err(x), + _ => panic!(), + } + } + } + } +} + +struct DialectRunningContext<'a> { + dialect: &'a Dialect, + py_int_allocator: &'a PyIntAllocator, +} + +impl DialectRunningContext<'_> { + pub fn invoke_py_obj( + &self, + obj: &PyObject, + allocator: &mut IntAllocator, + op_buf: ::AtomBuf, + args: &::Ptr, + max_cost: Cost, + ) -> Response<::Ptr> { + Python::with_gil(|py| { + let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); + let r = unwrap_or_eval_err( + self.py_int_allocator.py_for_native(py, args, allocator), + args, + "can't uncache", + )?; + let r1 = obj.call1(py, (op, r.to_object(py), max_cost)); + match r1 { + Err(pyerr) => { + let eval_err: PyResult> = + eval_err_for_pyerr(py, &pyerr, self.py_int_allocator, allocator); + let r: EvalErr = + unwrap_or_eval_err(eval_err, args, "unexpected exception")?; + Err(r) + } + Ok(o) => { + let pair: &PyTuple = unwrap_or_eval_err(o.extract(py), args, "expected tuple")?; + + let i0: u32 = + unwrap_or_eval_err(pair.get_item(0).extract(), args, "expected u32")?; + + let py_node: &PyCell = + unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; + + let r = self.py_int_allocator.native_for_py(py, py_node, allocator); + let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; + Ok(Reduction(i0 as Cost, node)) + } + } + }) + } +} + +impl OperatorHandler for DialectRunningContext<'_> { + fn op( + &self, + allocator: &mut IntAllocator, + o: ::AtomBuf, + argument_list: &::Ptr, + max_cost: Cost, + ) -> Response<::Ptr> { + let op = &allocator.buf(&o); + if op.len() == 1 { + if let Some(f) = self.dialect.u8_lookup[op[0] as usize] { + return f(allocator, argument_list.clone(), max_cost); + } + } + let op = op.to_owned(); + if let Some(op_fn) = self.dialect.native_u8_lookup.get(op) { + op_fn(allocator, argument_list.clone(), max_cost) + } else if let Some(op_fn) = self.dialect.python_u8_lookup.get(op) { + self.invoke_py_obj(op_fn, allocator, o, argument_list, max_cost) + } else { + if self.dialect.strict { + let buf = op.to_vec(); + let op_arg = allocator.new_atom(&buf)?; + err(op_arg, "unimplemented operator") + } else { + op_unknown(allocator, o, argument_list.clone(), max_cost) + } + } + } +} + +/// turn a `PyErr` into an `EvalErr

` if at all possible +/// otherwise, return a `PyErr` +fn eval_err_for_pyerr<'p>( + py: Python<'p>, + pyerr: &PyErr, + py_int_allocator: &'p PyIntAllocator, + allocator: &mut IntAllocator, +) -> PyResult> { + let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; + let arg0: &PyString = args.get_item(0).extract()?; + let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; + let node: i32 = py_int_allocator.native_for_py(py, sexp, allocator)?; + let s: String = arg0.to_str()?.to_string(); + Ok(EvalErr(node, s)) +} + +fn unwrap_or_eval_err(obj: PyResult, err_node: &P, msg: &str) -> Result> +where + P: Clone, +{ + match obj { + Err(_py_err) => Err(EvalErr(err_node.clone(), msg.to_string())), + Ok(o) => Ok(o), + } +} + +fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult { + let ctx: &PyDict = PyDict::new(py); + ctx.set_item("msg", msg)?; + ctx.set_item("sexp", sexp)?; + let r = py.run( + "from clvm.EvalError import EvalError; raise EvalError(msg, sexp)", + None, + Some(ctx), + ); + match r { + Err(x) => Err(x), + Ok(_) => Ok(ctx.into()), + } +} diff --git a/src/py/f_table.rs b/src/py/f_table.rs index 3422cec6..6e0bc098 100644 --- a/src/py/f_table.rs +++ b/src/py/f_table.rs @@ -10,7 +10,7 @@ use crate::more_ops::{ }; use crate::reduction::Response; -type OpFn = fn(&mut T, ::Ptr, Cost) -> Response<::Ptr>; +pub type OpFn = fn(&mut T, ::Ptr, Cost) -> Response<::Ptr>; pub type FLookup = [Option>; 256]; diff --git a/src/py/mod.rs b/src/py/mod.rs index 71c829c6..39798d2a 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -1,5 +1,7 @@ pub mod api; +pub mod dialect; pub mod f_table; +pub mod native_op; pub mod op_fn; pub mod py_int_allocator; pub mod py_int_node; diff --git a/src/py/native_op.rs b/src/py/native_op.rs new file mode 100644 index 00000000..3d8a0adc --- /dev/null +++ b/src/py/native_op.rs @@ -0,0 +1,18 @@ +use pyo3::prelude::pyclass; + +use crate::int_allocator::IntAllocator; + +use super::f_table::OpFn; + +//type OpFn = fn(&mut T, ::Ptr, Cost) -> Response<::Ptr>; + +#[pyclass] +pub struct NativeOp { + pub op: OpFn, +} + +impl NativeOp { + pub fn new(op: OpFn) -> Self { + Self { op } + } +} From 33cad2a8ce6147c5ade2dbf9d9b8b8d0d0da446f Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 25 Mar 2021 16:35:58 -0700 Subject: [PATCH 38/76] Fix clippy. --- src/py/dialect.rs | 16 +++++++--------- src/py/py_int_allocator.rs | 6 +++--- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 0084a3dc..a6c61086 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -179,22 +179,20 @@ impl OperatorHandler for DialectRunningContext<'_> { let op = &allocator.buf(&o); if op.len() == 1 { if let Some(f) = self.dialect.u8_lookup[op[0] as usize] { - return f(allocator, argument_list.clone(), max_cost); + return f(allocator, *argument_list, max_cost); } } let op = op.to_owned(); if let Some(op_fn) = self.dialect.native_u8_lookup.get(op) { - op_fn(allocator, argument_list.clone(), max_cost) + op_fn(allocator, *argument_list, max_cost) } else if let Some(op_fn) = self.dialect.python_u8_lookup.get(op) { self.invoke_py_obj(op_fn, allocator, o, argument_list, max_cost) + } else if self.dialect.strict { + let buf = op.to_vec(); + let op_arg = allocator.new_atom(&buf)?; + err(op_arg, "unimplemented operator") } else { - if self.dialect.strict { - let buf = op.to_vec(); - let op_arg = allocator.new_atom(&buf)?; - err(op_arg, "unimplemented operator") - } else { - op_unknown(allocator, o, argument_list.clone(), max_cost) - } + op_unknown(allocator, o, *argument_list, max_cost) } } } diff --git a/src/py/py_int_allocator.rs b/src/py/py_int_allocator.rs index e6c351b5..23bd1f18 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_int_allocator.rs @@ -18,13 +18,13 @@ pub struct PyIntAllocator { impl PyIntAllocator { pub fn new(py: Python) -> PyResult<&PyCell> { - Ok(PyCell::new( + PyCell::new( py, PyIntAllocator { arena: RefCell::new(IntAllocator::default()), cache: py.eval("dict()", None, None)?.to_object(py), }, - )?) + ) } pub fn allocator(&self) -> RefMut { @@ -193,7 +193,7 @@ impl PyIntAllocator { allocator: &mut IntAllocator, ) -> PyResult<&'p PyCell> { self.from_native_to_py_cache(py, ptr) - .or_else(|_err| Ok(self.populate_python(py, ptr, allocator)?)) + .or_else(|_err| self.populate_python(py, ptr, allocator)) } } From 8bf20a1b19205b6b8c27587d86804f86dcd3d825 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 25 Mar 2021 18:29:39 -0700 Subject: [PATCH 39/76] First crack at `Dialect` actually working! --- src/py/api.rs | 4 ++-- src/py/dialect.rs | 46 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 3924c81c..96e4a851 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -6,7 +6,7 @@ use pyo3::types::{PyBytes, PyDict, PyString}; use pyo3::wrap_pyfunction; use pyo3::PyObject; -use super::dialect::Dialect; +use super::dialect::{Dialect, __pyo3_get_function_native_opcodes_dict}; use super::py_int_allocator::PyIntAllocator; use super::py_int_node::PyIntNode; use super::py_node::PyNode; @@ -81,6 +81,7 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { //m.add_class::()?; m.add_class::()?; + m.add_function(wrap_pyfunction!(native_opcodes_dict, m)?)?; m.add_class::()?; m.add_class::()?; @@ -88,7 +89,6 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { //m.add_function(wrap_pyfunction!(serialized_length, m)?)?; m.add_class::()?; - //m.add_function(wrap_pyfunction!(raise_eval_error, m)?)?; Ok(()) } diff --git a/src/py/dialect.rs b/src/py/dialect.rs index a6c61086..53a94708 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -1,7 +1,7 @@ use std::cell::RefMut; use std::collections::HashMap; -use pyo3::prelude::{pyclass, pymethods}; +use pyo3::prelude::{pyclass, pyfunction, pymethods}; use pyo3::types::{PyBytes, PyDict, PyString, PyTuple}; use pyo3::{PyCell, PyErr, PyObject, PyRef, PyResult, Python, ToPyObject}; @@ -237,3 +237,47 @@ fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult Ok(ctx.into()), } } + +use crate::core_ops::*; +use crate::more_ops::*; + +#[pyfunction] +pub fn native_opcodes_dict(py: Python) -> PyResult { + let opcode_lookup: [(OpFn, &str); 30] = [ + (op_if, "op_if"), + (op_cons, "op_cons"), + (op_first, "op_first"), + (op_rest, "op_rest"), + (op_listp, "op_listp"), + (op_raise, "op_raise"), + (op_eq, "op_eq"), + (op_sha256, "op_sha256"), + (op_add, "op_add"), + (op_subtract, "op_subtract"), + (op_multiply, "op_multiply"), + (op_divmod, "op_divmod"), + (op_substr, "op_substr"), + (op_strlen, "op_strlen"), + (op_point_add, "op_point_add"), + (op_pubkey_for_exp, "op_pubkey_for_exp"), + (op_concat, "op_concat"), + (op_gr, "op_gr"), + (op_gr_bytes, "op_gr_bytes"), + (op_logand, "op_logand"), + (op_logior, "op_logior"), + (op_logxor, "op_logxor"), + (op_lognot, "op_lognot"), + (op_ash, "op_ash"), + (op_lsh, "op_lsh"), + (op_not, "op_not"), + (op_any, "op_any"), + (op_all, "op_all"), + (op_softfork, "op_softfork"), + (op_div, "op_div"), + ]; + let r = PyDict::new(py); + for (f, name) in opcode_lookup.iter() { + r.set_item(name, PyCell::new(py, NativeOp::new(*f))?)?; + } + Ok(r.to_object(py)) +} From f00ea74692784bdadfa81ea500cba07147aab81e Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 26 Mar 2021 17:32:37 -0700 Subject: [PATCH 40/76] Rename checkpoint. --- src/py/api.rs | 55 +++++++++++---------- src/py/dialect.rs | 30 +++++------ src/py/mod.rs | 4 +- src/py/native_clvm_object.rs | 41 +++++++++++++++ src/py/op_fn.rs | 18 +++---- src/py/{py_int_allocator.rs => py_arena.rs} | 6 +-- src/py/py_int_node.rs | 44 ----------------- src/py/run_program.rs | 8 +-- 8 files changed, 103 insertions(+), 103 deletions(-) create mode 100644 src/py/native_clvm_object.rs rename src/py/{py_int_allocator.rs => py_arena.rs} (98%) delete mode 100644 src/py/py_int_node.rs diff --git a/src/py/api.rs b/src/py/api.rs index 96e4a851..e86402c3 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -7,10 +7,13 @@ use pyo3::wrap_pyfunction; use pyo3::PyObject; use super::dialect::{Dialect, __pyo3_get_function_native_opcodes_dict}; -use super::py_int_allocator::PyIntAllocator; -use super::py_int_node::PyIntNode; +use super::native_clvm_object::NativeClvmObject; +use super::py_arena::PyArena; use super::py_node::PyNode; -use super::run_program::{__pyo3_get_function_deserialize_and_run_program, STRICT_MODE}; +use super::run_program::{ + __pyo3_get_function_deserialize_and_run_program, __pyo3_get_function_serialized_length, + STRICT_MODE, +}; use crate::int_allocator::IntAllocator; @@ -37,31 +40,31 @@ fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult( py: Python<'p>, blob: &[u8], - py_int_allocator: &PyCell, -) -> PyResult { + arena: &PyCell, +) -> PyResult { let ptr = { - let py_int_allocator: PyRef = py_int_allocator.borrow(); - let allocator: &mut IntAllocator = &mut py_int_allocator.allocator() as &mut IntAllocator; + let arena: PyRef = arena.borrow(); + let allocator: &mut IntAllocator = &mut arena.allocator() as &mut IntAllocator; node_from_bytes(allocator, blob)? }; - Ok(PyIntNode::new(py, py_int_allocator, ptr)) + Ok(NativeClvmObject::new(py, arena, ptr)) } #[pyfunction] -fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult { - let py_int_allocator = PyIntAllocator::new(py)?; - deserialize_from_bytes_for_allocator(py, blob, &py_int_allocator) +fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult { + let arena = PyArena::new(py)?; + deserialize_from_bytes_for_allocator(py, blob, &arena) } use crate::node::Node; #[pyfunction] fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { - let py_int_allocator = PyIntAllocator::new(py)?.borrow(); - let mut allocator_refcell: RefMut = py_int_allocator.allocator(); + let arena = PyArena::new(py)?.borrow(); + let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let ptr = py_int_allocator.native_for_py(py, sexp, allocator)?; + let ptr = arena.native_for_py(py, sexp, allocator)?; let node = Node::new(allocator, ptr); let s: Vec = node_to_bytes(&node)?; @@ -71,6 +74,10 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p /// This module is a python module implemented in Rust. #[pymodule] fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(py_run_program, m)?)?; m.add_function(wrap_pyfunction!(deserialize_from_bytes, m)?)?; m.add_function(wrap_pyfunction!(deserialize_from_bytes_for_allocator, m)?)?; @@ -79,17 +86,13 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; m.add("STRICT_MODE", STRICT_MODE)?; - //m.add_class::()?; m.add_class::()?; m.add_function(wrap_pyfunction!(native_opcodes_dict, m)?)?; - m.add_class::()?; - m.add_class::()?; + m.add_function(wrap_pyfunction!(serialized_length, m)?)?; - //m.add_function(wrap_pyfunction!(serialized_length, m)?)?; m.add_class::()?; - Ok(()) } @@ -108,13 +111,13 @@ pub fn py_run_program<'p>( opcode_lookup_by_name: HashMap>, py_callback: PyObject, ) -> PyResult<(Cost, PyObject)> { - let py_int_allocator = PyIntAllocator::new(py)?.borrow(); - let mut allocator_refcell: RefMut = py_int_allocator.allocator(); + let arena = PyArena::new(py)?.borrow(); + let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let op_lookup = PyOperatorHandler::new(opcode_lookup_by_name, py_callback, &py_int_allocator)?; - let program = py_int_allocator.native_for_py(py, program, allocator)?; - let args = py_int_allocator.native_for_py(py, args, allocator)?; + let op_lookup = PyOperatorHandler::new(opcode_lookup_by_name, py_callback, &arena)?; + let program = arena.native_for_py(py, program, allocator)?; + let args = arena.native_for_py(py, args, allocator)?; let r: Result, EvalErr> = crate::run_program::run_program( allocator, &program, &args, quote_kw, apply_kw, max_cost, &op_lookup, None, @@ -122,11 +125,11 @@ pub fn py_run_program<'p>( match r { Ok(reduction) => { - let r = py_int_allocator.py_for_native(py, &reduction.1, allocator)?; + let r = arena.py_for_native(py, &reduction.1, allocator)?; Ok((reduction.0, r.to_object(py))) } Err(eval_err) => { - let node: PyObject = py_int_allocator + let node: PyObject = arena .py_for_native(py, &eval_err.0, allocator)? .to_object(py); let s: String = eval_err.1; diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 53a94708..b3b8868e 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -19,7 +19,7 @@ use crate::run_program::OperatorHandler; use super::f_table::FLookup; use super::f_table::OpFn; use super::native_op::NativeOp; -use super::py_int_allocator::PyIntAllocator; +use super::py_arena::PyArena; use super::py_node::PyNode; #[pyclass] @@ -75,18 +75,18 @@ impl Dialect { args: &PyCell, max_cost: Cost, ) -> PyResult<(Cost, PyObject)> { - let py_int_allocator = PyIntAllocator::new(py)?; - let py_int_allocator = &py_int_allocator.borrow() as &PyIntAllocator; + let arena = PyArena::new(py)?; + let arena = &arena.borrow() as &PyArena; let drc = DialectRunningContext { dialect: self, - py_int_allocator, + arena, }; - let mut allocator_refcell: RefMut = py_int_allocator.allocator(); + let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let program = py_int_allocator.native_for_py(py, program, allocator)?; - let args = py_int_allocator.native_for_py(py, args, allocator)?; + let program = arena.native_for_py(py, program, allocator)?; + let args = arena.native_for_py(py, args, allocator)?; let r: Result, EvalErr> = crate::run_program::run_program( allocator, @@ -101,11 +101,11 @@ impl Dialect { match r { Ok(reduction) => { - let r = py_int_allocator.py_for_native(py, &reduction.1, allocator)?; + let r = arena.py_for_native(py, &reduction.1, allocator)?; Ok((reduction.0, r.to_object(py))) } Err(eval_err) => { - let node: PyObject = py_int_allocator + let node: PyObject = arena .py_for_native(py, &eval_err.0, allocator)? .to_object(py); let s: String = eval_err.1; @@ -122,7 +122,7 @@ impl Dialect { struct DialectRunningContext<'a> { dialect: &'a Dialect, - py_int_allocator: &'a PyIntAllocator, + arena: &'a PyArena, } impl DialectRunningContext<'_> { @@ -137,7 +137,7 @@ impl DialectRunningContext<'_> { Python::with_gil(|py| { let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); let r = unwrap_or_eval_err( - self.py_int_allocator.py_for_native(py, args, allocator), + self.arena.py_for_native(py, args, allocator), args, "can't uncache", )?; @@ -145,7 +145,7 @@ impl DialectRunningContext<'_> { match r1 { Err(pyerr) => { let eval_err: PyResult> = - eval_err_for_pyerr(py, &pyerr, self.py_int_allocator, allocator); + eval_err_for_pyerr(py, &pyerr, self.arena, allocator); let r: EvalErr = unwrap_or_eval_err(eval_err, args, "unexpected exception")?; Err(r) @@ -159,7 +159,7 @@ impl DialectRunningContext<'_> { let py_node: &PyCell = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = self.py_int_allocator.native_for_py(py, py_node, allocator); + let r = self.arena.native_for_py(py, py_node, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } @@ -202,13 +202,13 @@ impl OperatorHandler for DialectRunningContext<'_> { fn eval_err_for_pyerr<'p>( py: Python<'p>, pyerr: &PyErr, - py_int_allocator: &'p PyIntAllocator, + arena: &'p PyArena, allocator: &mut IntAllocator, ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = py_int_allocator.native_for_py(py, sexp, allocator)?; + let node: i32 = arena.native_for_py(py, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) } diff --git a/src/py/mod.rs b/src/py/mod.rs index 39798d2a..af01a2d5 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -1,10 +1,10 @@ pub mod api; pub mod dialect; pub mod f_table; +pub mod native_clvm_object; pub mod native_op; pub mod op_fn; -pub mod py_int_allocator; -pub mod py_int_node; +pub mod py_arena; pub mod py_node; pub mod py_view; pub mod run_program; diff --git a/src/py/native_clvm_object.rs b/src/py/native_clvm_object.rs new file mode 100644 index 00000000..282fae59 --- /dev/null +++ b/src/py/native_clvm_object.rs @@ -0,0 +1,41 @@ +use std::cell::RefMut; + +use pyo3::prelude::{pyclass, pymethods}; +use pyo3::PyCell; +use pyo3::PyObject; +use pyo3::PyRef; +use pyo3::PyResult; +use pyo3::Python; +use pyo3::ToPyObject; + +use crate::int_allocator::IntAllocator; + +use super::py_arena::PyArena; +use super::py_node::PyNode; + +#[pyclass(weakref, subclass)] +pub struct NativeClvmObject { + arena: PyObject, + ptr: i32, +} + +impl NativeClvmObject { + pub fn new(py: Python, arena: &PyCell, ptr: i32) -> Self { + let arena = arena.to_object(py); + Self { arena, ptr } + } +} + +#[pymethods] +impl NativeClvmObject { + fn clvm_object<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { + let arena: PyRef = self.arena.extract(py)?; + let mut allocator: RefMut = arena.allocator(); + arena.py_for_native(py, &self.ptr, &mut allocator) + } + + #[getter(arena)] + fn arena<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { + self.arena.extract(py) + } +} diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 1e6c86cf..382fbc03 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -17,26 +17,26 @@ use crate::reduction::{EvalErr, Reduction, Response}; use crate::run_program::OperatorHandler; use super::f_table::{f_lookup_for_hashmap, FLookup}; -use super::py_int_allocator::PyIntAllocator; +use super::py_arena::PyArena; use super::py_node::PyNode; pub struct PyOperatorHandler<'p> { native_lookup: FLookup, py_callable: PyObject, - py_int_allocator: &'p PyIntAllocator, + arena: &'p PyArena, } impl<'p> PyOperatorHandler<'p> { pub fn new( opcode_lookup_by_name: HashMap>, py_callable: PyObject, - py_int_allocator: &'p PyIntAllocator, + arena: &'p PyArena, ) -> PyResult { let native_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); Ok(PyOperatorHandler { native_lookup, py_callable, - py_int_allocator, + arena, }) } @@ -51,7 +51,7 @@ impl<'p> PyOperatorHandler<'p> { Python::with_gil(|py| { let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); let r = unwrap_or_eval_err( - self.py_int_allocator.py_for_native(py, args, allocator), + self.arena.py_for_native(py, args, allocator), args, "can't uncache", )?; @@ -59,7 +59,7 @@ impl<'p> PyOperatorHandler<'p> { match r1 { Err(pyerr) => { let eval_err: PyResult> = - eval_err_for_pyerr(py, &pyerr, self.py_int_allocator, allocator); + eval_err_for_pyerr(py, &pyerr, self.arena, allocator); let r: EvalErr = unwrap_or_eval_err(eval_err, args, "unexpected exception")?; Err(r) @@ -73,7 +73,7 @@ impl<'p> PyOperatorHandler<'p> { let py_node: &PyCell = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = self.py_int_allocator.native_for_py(py, py_node, allocator); + let r = self.arena.native_for_py(py, py_node, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } @@ -106,13 +106,13 @@ impl OperatorHandler for PyOperatorHandler<'_> { fn eval_err_for_pyerr<'p>( py: Python<'p>, pyerr: &PyErr, - py_int_allocator: &'p PyIntAllocator, + arena: &'p PyArena, allocator: &mut IntAllocator, ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = py_int_allocator.native_for_py(py, sexp, allocator)?; + let node: i32 = arena.native_for_py(py, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) } diff --git a/src/py/py_int_allocator.rs b/src/py/py_arena.rs similarity index 98% rename from src/py/py_int_allocator.rs rename to src/py/py_arena.rs index 23bd1f18..2bb414e9 100644 --- a/src/py/py_int_allocator.rs +++ b/src/py/py_arena.rs @@ -11,16 +11,16 @@ use super::py_node::PyNode; use super::py_view::PyView; #[pyclass(subclass, unsendable)] -pub struct PyIntAllocator { +pub struct PyArena { arena: RefCell, cache: PyObject, } -impl PyIntAllocator { +impl PyArena { pub fn new(py: Python) -> PyResult<&PyCell> { PyCell::new( py, - PyIntAllocator { + PyArena { arena: RefCell::new(IntAllocator::default()), cache: py.eval("dict()", None, None)?.to_object(py), }, diff --git a/src/py/py_int_node.rs b/src/py/py_int_node.rs deleted file mode 100644 index ae3b872b..00000000 --- a/src/py/py_int_node.rs +++ /dev/null @@ -1,44 +0,0 @@ -use std::cell::RefMut; - -use pyo3::prelude::{pyclass, pymethods}; -use pyo3::PyCell; -use pyo3::PyObject; -use pyo3::PyRef; -use pyo3::PyResult; -use pyo3::Python; -use pyo3::ToPyObject; - -use crate::int_allocator::IntAllocator; - -use super::py_int_allocator::PyIntAllocator; -use super::py_node::PyNode; - -#[pyclass(weakref, subclass)] -pub struct PyIntNode { - py_int_allocator: PyObject, - ptr: i32, -} - -impl PyIntNode { - pub fn new(py: Python, py_int_allocator: &PyCell, ptr: i32) -> Self { - let py_int_allocator = py_int_allocator.to_object(py); - Self { - py_int_allocator, - ptr, - } - } -} - -#[pymethods] -impl PyIntNode { - fn clvm_object<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { - let py_int_allocator: PyRef = self.py_int_allocator.extract(py)?; - let mut allocator: RefMut = py_int_allocator.allocator(); - py_int_allocator.py_for_native(py, &self.ptr, &mut allocator) - } - - #[getter(arena)] - fn arena<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { - self.py_int_allocator.extract(py) - } -} diff --git a/src/py/run_program.rs b/src/py/run_program.rs index b74c4cfb..a44c017b 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -12,7 +12,7 @@ use crate::run_program::{run_program, OperatorHandler}; use crate::serialize::{node_from_bytes, node_to_bytes, serialized_length_from_bytes}; use super::f_table::{f_lookup_for_hashmap, FLookup}; -use super::py_int_allocator::PyIntAllocator; +use super::py_arena::PyArena; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict}; @@ -60,8 +60,8 @@ pub fn deserialize_and_run_program( max_cost: Cost, flags: u32, ) -> PyResult<(Cost, PyObject)> { - let py_int_allocator = PyIntAllocator::new(py)?.borrow(); - let mut allocator_refcell: RefMut = py_int_allocator.allocator(); + let arena = PyArena::new(py)?.borrow(); + let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; let f_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); let strict: bool = (flags & STRICT_MODE) != 0; @@ -75,7 +75,7 @@ pub fn deserialize_and_run_program( match r { Ok(reduction) => Ok(( reduction.0, - py_int_allocator + arena .py_for_native(py, &reduction.1, allocator)? .to_object(py), )), From 8c4aee009fe83d684995d589c020900a5175fa04 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 26 Mar 2021 19:33:51 -0700 Subject: [PATCH 41/76] More rename. --- src/py/api.rs | 22 ++++++++---------- src/py/{py_node.rs => clvm_object.rs} | 12 +++++----- src/py/dialect.rs | 12 +++++----- src/py/mod.rs | 2 +- src/py/native_clvm_object.rs | 10 ++++---- src/py/op_fn.rs | 8 +++---- src/py/py_arena.rs | 33 +++++++++++++++------------ src/py/py_view.rs | 6 ++--- 8 files changed, 54 insertions(+), 51 deletions(-) rename src/py/{py_node.rs => clvm_object.rs} (92%) diff --git a/src/py/api.rs b/src/py/api.rs index e86402c3..b72a54a2 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -6,10 +6,10 @@ use pyo3::types::{PyBytes, PyDict, PyString}; use pyo3::wrap_pyfunction; use pyo3::PyObject; +use super::clvm_object::CLVMObject; use super::dialect::{Dialect, __pyo3_get_function_native_opcodes_dict}; -use super::native_clvm_object::NativeClvmObject; +use super::native_clvm_object::NativeCLVMObject; use super::py_arena::PyArena; -use super::py_node::PyNode; use super::run_program::{ __pyo3_get_function_deserialize_and_run_program, __pyo3_get_function_serialized_length, STRICT_MODE, @@ -41,17 +41,17 @@ fn deserialize_from_bytes_for_allocator<'p>( py: Python<'p>, blob: &[u8], arena: &PyCell, -) -> PyResult { +) -> PyResult { let ptr = { let arena: PyRef = arena.borrow(); let allocator: &mut IntAllocator = &mut arena.allocator() as &mut IntAllocator; node_from_bytes(allocator, blob)? }; - Ok(NativeClvmObject::new(py, arena, ptr)) + Ok(NativeCLVMObject::new(py, arena, ptr)) } #[pyfunction] -fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult { +fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult { let arena = PyArena::new(py)?; deserialize_from_bytes_for_allocator(py, blob, &arena) } @@ -59,7 +59,7 @@ fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult use crate::node::Node; #[pyfunction] -fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { +fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { let arena = PyArena::new(py)?.borrow(); let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; @@ -75,8 +75,8 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p #[pymodule] fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; - m.add_class::()?; - m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_function(wrap_pyfunction!(py_run_program, m)?)?; m.add_function(wrap_pyfunction!(deserialize_from_bytes, m)?)?; @@ -91,8 +91,6 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(serialized_length, m)?)?; - m.add_class::()?; - Ok(()) } @@ -103,8 +101,8 @@ use crate::reduction::{EvalErr, Reduction}; #[allow(clippy::too_many_arguments)] pub fn py_run_program<'p>( py: Python<'p>, - program: &PyCell, - args: &PyCell, + program: &PyCell, + args: &PyCell, quote_kw: u8, apply_kw: u8, max_cost: Cost, diff --git a/src/py/py_node.rs b/src/py/clvm_object.rs similarity index 92% rename from src/py/py_node.rs rename to src/py/clvm_object.rs index 314c4635..649e2a04 100644 --- a/src/py/py_node.rs +++ b/src/py/clvm_object.rs @@ -4,18 +4,18 @@ use pyo3::types::{PyBytes, PyTuple, PyType}; use super::py_view::PyView; #[pyclass(weakref, subclass)] -pub struct PyNode { +pub struct CLVMObject { pub py_view: PyView, } -impl PyNode { +impl CLVMObject { pub fn new(py: Python, py_view: PyView) -> PyResult<&PyCell> { - PyCell::new(py, PyNode { py_view }) + PyCell::new(py, CLVMObject { py_view }) } } #[pymethods] -impl PyNode { +impl CLVMObject { #[new] fn new_obj(py: Python, obj: &PyAny) -> PyResult { Ok(if let Ok(tuple) = obj.extract() { @@ -38,8 +38,8 @@ impl PyNode { fn new_pair<'p>( _cls: &PyType, py: Python<'p>, - p1: &PyCell, - p2: &PyCell, + p1: &PyCell, + p2: &PyCell, ) -> PyResult<&'p PyCell> { let tuple = PyTuple::new(py, &[p1, p2]); let py_view = PyView::new_pair(py, tuple)?; diff --git a/src/py/dialect.rs b/src/py/dialect.rs index b3b8868e..3257342c 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -16,11 +16,11 @@ use crate::reduction::Reduction; use crate::reduction::Response; use crate::run_program::OperatorHandler; +use super::clvm_object::CLVMObject; use super::f_table::FLookup; use super::f_table::OpFn; use super::native_op::NativeOp; use super::py_arena::PyArena; -use super::py_node::PyNode; #[pyclass] pub struct Dialect { @@ -71,8 +71,8 @@ impl Dialect { pub fn run_program<'p>( &self, py: Python<'p>, - program: &PyCell, - args: &PyCell, + program: &PyCell, + args: &PyCell, max_cost: Cost, ) -> PyResult<(Cost, PyObject)> { let arena = PyArena::new(py)?; @@ -156,10 +156,10 @@ impl DialectRunningContext<'_> { let i0: u32 = unwrap_or_eval_err(pair.get_item(0).extract(), args, "expected u32")?; - let py_node: &PyCell = + let clvm_object: &PyCell = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = self.arena.native_for_py(py, py_node, allocator); + let r = self.arena.native_for_py(py, clvm_object, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } @@ -207,7 +207,7 @@ fn eval_err_for_pyerr<'p>( ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; - let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; + let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; let node: i32 = arena.native_for_py(py, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) diff --git a/src/py/mod.rs b/src/py/mod.rs index af01a2d5..76372843 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -1,10 +1,10 @@ pub mod api; +pub mod clvm_object; pub mod dialect; pub mod f_table; pub mod native_clvm_object; pub mod native_op; pub mod op_fn; pub mod py_arena; -pub mod py_node; pub mod py_view; pub mod run_program; diff --git a/src/py/native_clvm_object.rs b/src/py/native_clvm_object.rs index 282fae59..b8f845d4 100644 --- a/src/py/native_clvm_object.rs +++ b/src/py/native_clvm_object.rs @@ -10,16 +10,16 @@ use pyo3::ToPyObject; use crate::int_allocator::IntAllocator; +use super::clvm_object::CLVMObject; use super::py_arena::PyArena; -use super::py_node::PyNode; #[pyclass(weakref, subclass)] -pub struct NativeClvmObject { +pub struct NativeCLVMObject { arena: PyObject, ptr: i32, } -impl NativeClvmObject { +impl NativeCLVMObject { pub fn new(py: Python, arena: &PyCell, ptr: i32) -> Self { let arena = arena.to_object(py); Self { arena, ptr } @@ -27,8 +27,8 @@ impl NativeClvmObject { } #[pymethods] -impl NativeClvmObject { - fn clvm_object<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { +impl NativeCLVMObject { + fn clvm_object<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { let arena: PyRef = self.arena.extract(py)?; let mut allocator: RefMut = arena.allocator(); arena.py_for_native(py, &self.ptr, &mut allocator) diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 382fbc03..2964fa69 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -16,9 +16,9 @@ use crate::int_allocator::IntAllocator; use crate::reduction::{EvalErr, Reduction, Response}; use crate::run_program::OperatorHandler; +use super::clvm_object::CLVMObject; use super::f_table::{f_lookup_for_hashmap, FLookup}; use super::py_arena::PyArena; -use super::py_node::PyNode; pub struct PyOperatorHandler<'p> { native_lookup: FLookup, @@ -70,10 +70,10 @@ impl<'p> PyOperatorHandler<'p> { let i0: u32 = unwrap_or_eval_err(pair.get_item(0).extract(), args, "expected u32")?; - let py_node: &PyCell = + let clvm_object: &PyCell = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = self.arena.native_for_py(py, py_node, allocator); + let r = self.arena.native_for_py(py, clvm_object, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } @@ -111,7 +111,7 @@ fn eval_err_for_pyerr<'p>( ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; - let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; + let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; let node: i32 = arena.native_for_py(py, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index 2bb414e9..5c656918 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -7,7 +7,7 @@ use pyo3::types::{IntoPyDict, PyBytes, PyTuple}; use crate::allocator::{Allocator, SExp}; use crate::int_allocator::IntAllocator; -use super::py_node::PyNode; +use super::clvm_object::CLVMObject; use super::py_view::PyView; #[pyclass(subclass, unsendable)] @@ -34,7 +34,7 @@ impl PyArena { pub fn add( &self, py: Python, - obj: &PyCell, + obj: &PyCell, ptr: &::Ptr, ) -> PyResult<()> { let locals = [ @@ -52,7 +52,7 @@ impl PyArena { fn from_py_to_native_cache<'p>( &self, py: Python<'p>, - obj: &PyCell, + obj: &PyCell, ) -> PyResult<::Ptr> { let locals = [("cache", self.cache.clone()), ("key", obj.to_object(py))].into_py_dict(py); py.eval("cache.get(id(key))", None, Some(locals))?.extract() @@ -61,11 +61,11 @@ impl PyArena { fn populate_native( &self, py: Python, - obj: &PyCell, + obj: &PyCell, allocator: &mut IntAllocator, ) -> PyResult<::Ptr> { apply_to_tree(obj.to_object(py), move |obj| { - let node: &PyCell = obj.extract(py)?; + let node: &PyCell = obj.extract(py)?; // is it in cache yet? if self.from_py_to_native_cache(py, node).is_ok() { @@ -86,8 +86,8 @@ impl PyArena { PyView::Pair(pair) => { let pair: &PyAny = pair.clone().into_ref(py); let pair: &PyTuple = pair.extract()?; - let p0: &PyCell = pair.get_item(0).extract()?; - let p1: &PyCell = pair.get_item(1).extract()?; + let p0: &PyCell = pair.get_item(0).extract()?; + let p1: &PyCell = pair.get_item(1).extract()?; let ptr_0: PyResult = self.from_py_to_native_cache(py, p0); let ptr_1: PyResult = self.from_py_to_native_cache(py, p1); if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { @@ -107,7 +107,7 @@ impl PyArena { pub fn native_for_py( &self, py: Python, - obj: &PyCell, + obj: &PyCell, allocator: &mut IntAllocator, ) -> PyResult<::Ptr> { self.from_py_to_native_cache(py, obj) @@ -120,7 +120,7 @@ impl PyArena { &self, py: Python<'p>, ptr: &::Ptr, - ) -> PyResult<&'p PyCell> { + ) -> PyResult<&'p PyCell> { let locals = [("cache", self.cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); py.eval("cache[key]", None, Some(locals))?.extract() } @@ -130,7 +130,7 @@ impl PyArena { py: Python<'p>, ptr: &::Ptr, allocator: &mut IntAllocator, - ) -> PyResult<&'p PyCell> { + ) -> PyResult<&'p PyCell> { apply_to_tree(*ptr, move |ptr| { // is it in cache yet? if self.from_native_to_py_cache(py, &ptr).is_ok() { @@ -145,7 +145,11 @@ impl PyArena { // it's an atom, so we just populate cache directly let blob = allocator.buf(&a); let py_bytes = PyBytes::new(py, blob); - self.add(py, PyNode::new(py, PyView::new_atom(py, py_bytes))?, &ptr)?; + self.add( + py, + CLVMObject::new(py, PyView::new_atom(py, py_bytes))?, + &ptr, + )?; Ok(None) } SExp::Pair(ptr_1, ptr_2) => { @@ -167,10 +171,11 @@ impl PyArena { // the children are in the cache, create new node & populate cache with it Ok(tuple) => { - let (p1, p2): (&PyCell, &PyCell) = tuple.extract()?; + let (p1, p2): (&PyCell, &PyCell) = + tuple.extract()?; self.add( py, - PyNode::new( + CLVMObject::new( py, PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?, )?, @@ -191,7 +196,7 @@ impl PyArena { py: Python<'p>, ptr: &::Ptr, allocator: &mut IntAllocator, - ) -> PyResult<&'p PyCell> { + ) -> PyResult<&'p PyCell> { self.from_native_to_py_cache(py, ptr) .or_else(|_err| self.populate_python(py, ptr, allocator)) } diff --git a/src/py/py_view.rs b/src/py/py_view.rs index 15319ac9..9c025b52 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -2,7 +2,7 @@ use pyo3::pycell::PyCell; use pyo3::types::{PyBytes, PyTuple}; use pyo3::{PyObject, PyResult, Python, ToPyObject}; -use super::py_node::PyNode; +use super::clvm_object::CLVMObject; pub enum PyView { Atom(PyObject), @@ -18,8 +18,8 @@ impl PyView { if pair.len() != 2 { py.eval("raise ValueError('new_pair requires 2-tuple')", None, None)?; } - let _p0: &PyCell = pair.get_item(0).extract()?; - let _p1: &PyCell = pair.get_item(1).extract()?; + let _p0: &PyCell = pair.get_item(0).extract()?; + let _p1: &PyCell = pair.get_item(1).extract()?; Ok(PyView::Pair(pair.to_object(py))) } From a2065c1b6abedec6625aea7828ab0537b34a944d Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 1 Apr 2021 15:44:31 -0700 Subject: [PATCH 42/76] First gross crack at `MultiOpFnE`. --- src/py/dialect.rs | 72 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 3257342c..b7aff7af 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use pyo3::prelude::{pyclass, pyfunction, pymethods}; use pyo3::types::{PyBytes, PyDict, PyString, PyTuple}; -use pyo3::{PyCell, PyErr, PyObject, PyRef, PyResult, Python, ToPyObject}; +use pyo3::{FromPyObject, PyCell, PyErr, PyObject, PyRef, PyResult, Python, ToPyObject}; use crate::allocator::Allocator; use crate::cost::Cost; @@ -22,6 +22,62 @@ use super::f_table::OpFn; use super::native_op::NativeOp; use super::py_arena::PyArena; +pub type MultiOpFn = fn( + &mut T, + ::AtomBuf, + ::Ptr, + Cost, +) -> Response<::Ptr>; + +#[derive(Clone)] +pub enum MultiOpFnE { + Python(PyObject), + Rust( + fn( + &mut T, + ::AtomBuf, + ::Ptr, + Cost, + ) -> Response<::Ptr>, + ), +} + +impl MultiOpFnE { + pub fn invoke( + &self, + allocator: &mut T, + o: ::AtomBuf, + args: ::Ptr, + max_cost: Cost, + ) -> Response<::Ptr> { + match self { + Self::Python(o) => { + panic!("oops") + } + Self::Rust(f) => f(allocator, o, args, max_cost), + } + } +} + +impl<'source> FromPyObject<'source> for MultiOpFnE { + fn extract(obj: &'source pyo3::PyAny) -> std::result::Result { + let v: PyResult = obj.extract(); + if let Ok(v) = v { + Ok(Self::Rust(if v == 0 { + op_unknown + } else { + |_a, _b, op, _d| { + //let buf = op.to_vec(); + //let op_arg = allocator.new_atom(&buf)?; + err(op, "unimplemented operator") + } + })) + } else { + Ok(Self::Python(obj.into())) + } + } +} + #[pyclass] pub struct Dialect { quote_kw: u8, @@ -29,7 +85,7 @@ pub struct Dialect { u8_lookup: FLookup, python_u8_lookup: HashMap, PyObject>, native_u8_lookup: HashMap, OpFn>, - strict: bool, + unknown_op_callback: MultiOpFnE, } #[pymethods] @@ -40,7 +96,7 @@ impl Dialect { quote_kw: u8, apply_kw: u8, op_table: HashMap, PyObject>, - strict: bool, + unknown_op_callback: MultiOpFnE, ) -> PyResult { let mut u8_lookup = [None; 256]; let mut python_u8_lookup = HashMap::new(); @@ -64,7 +120,7 @@ impl Dialect { u8_lookup, python_u8_lookup, native_u8_lookup, - strict, + unknown_op_callback, }) } @@ -187,12 +243,10 @@ impl OperatorHandler for DialectRunningContext<'_> { op_fn(allocator, *argument_list, max_cost) } else if let Some(op_fn) = self.dialect.python_u8_lookup.get(op) { self.invoke_py_obj(op_fn, allocator, o, argument_list, max_cost) - } else if self.dialect.strict { - let buf = op.to_vec(); - let op_arg = allocator.new_atom(&buf)?; - err(op_arg, "unimplemented operator") } else { - op_unknown(allocator, o, *argument_list, max_cost) + self.dialect + .unknown_op_callback + .invoke(allocator, o, *argument_list, max_cost) } } } From c58ee9a025d1661249693f2431cf499ae588d012 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 9 Apr 2021 15:53:58 -0700 Subject: [PATCH 43/76] Rename to `ArenaObject`. --- src/py/api.rs | 10 +++++----- src/py/{native_clvm_object.rs => arena_object.rs} | 6 +++--- src/py/mod.rs | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) rename src/py/{native_clvm_object.rs => arena_object.rs} (92%) diff --git a/src/py/api.rs b/src/py/api.rs index b72a54a2..35155d5d 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -6,9 +6,9 @@ use pyo3::types::{PyBytes, PyDict, PyString}; use pyo3::wrap_pyfunction; use pyo3::PyObject; +use super::arena_object::ArenaObject; use super::clvm_object::CLVMObject; use super::dialect::{Dialect, __pyo3_get_function_native_opcodes_dict}; -use super::native_clvm_object::NativeCLVMObject; use super::py_arena::PyArena; use super::run_program::{ __pyo3_get_function_deserialize_and_run_program, __pyo3_get_function_serialized_length, @@ -41,17 +41,17 @@ fn deserialize_from_bytes_for_allocator<'p>( py: Python<'p>, blob: &[u8], arena: &PyCell, -) -> PyResult { +) -> PyResult { let ptr = { let arena: PyRef = arena.borrow(); let allocator: &mut IntAllocator = &mut arena.allocator() as &mut IntAllocator; node_from_bytes(allocator, blob)? }; - Ok(NativeCLVMObject::new(py, arena, ptr)) + Ok(ArenaObject::new(py, arena, ptr)) } #[pyfunction] -fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult { +fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult { let arena = PyArena::new(py)?; deserialize_from_bytes_for_allocator(py, blob, &arena) } @@ -75,7 +75,7 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult #[pymodule] fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_function(wrap_pyfunction!(py_run_program, m)?)?; diff --git a/src/py/native_clvm_object.rs b/src/py/arena_object.rs similarity index 92% rename from src/py/native_clvm_object.rs rename to src/py/arena_object.rs index b8f845d4..f1bb19dc 100644 --- a/src/py/native_clvm_object.rs +++ b/src/py/arena_object.rs @@ -14,12 +14,12 @@ use super::clvm_object::CLVMObject; use super::py_arena::PyArena; #[pyclass(weakref, subclass)] -pub struct NativeCLVMObject { +pub struct ArenaObject { arena: PyObject, ptr: i32, } -impl NativeCLVMObject { +impl ArenaObject { pub fn new(py: Python, arena: &PyCell, ptr: i32) -> Self { let arena = arena.to_object(py); Self { arena, ptr } @@ -27,7 +27,7 @@ impl NativeCLVMObject { } #[pymethods] -impl NativeCLVMObject { +impl ArenaObject { fn clvm_object<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { let arena: PyRef = self.arena.extract(py)?; let mut allocator: RefMut = arena.allocator(); diff --git a/src/py/mod.rs b/src/py/mod.rs index 76372843..a3094bc7 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -1,8 +1,8 @@ pub mod api; +pub mod arena_object; pub mod clvm_object; pub mod dialect; pub mod f_table; -pub mod native_clvm_object; pub mod native_op; pub mod op_fn; pub mod py_arena; From bd9ec944127b17a96541a1452322bc81523993a0 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 9 Apr 2021 16:59:55 -0700 Subject: [PATCH 44/76] Deprecate `STRICT` and be explicit about fallback function. --- src/py/api.rs | 72 +++++++++++++++++++++++++++++------ src/py/dialect.rs | 87 ++++++++++--------------------------------- src/py/run_program.rs | 12 +++--- 3 files changed, 85 insertions(+), 86 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 35155d5d..a4ea9c53 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -6,19 +6,66 @@ use pyo3::types::{PyBytes, PyDict, PyString}; use pyo3::wrap_pyfunction; use pyo3::PyObject; +use crate::core_ops::*; +use crate::cost::Cost; +use crate::err_utils::err; +use crate::int_allocator::IntAllocator; +use crate::more_ops::*; +use crate::node::Node; +use crate::reduction::{EvalErr, Reduction}; +use crate::serialize::{node_from_bytes, node_to_bytes}; + use super::arena_object::ArenaObject; use super::clvm_object::CLVMObject; -use super::dialect::{Dialect, __pyo3_get_function_native_opcodes_dict}; +use super::dialect::{Dialect, PyMultiOpFn}; +use super::f_table::OpFn; +use super::native_op::NativeOp; +use super::op_fn::PyOperatorHandler; use super::py_arena::PyArena; use super::run_program::{ __pyo3_get_function_deserialize_and_run_program, __pyo3_get_function_serialized_length, - STRICT_MODE, }; -use crate::int_allocator::IntAllocator; - -use crate::cost::Cost; -use crate::serialize::{node_from_bytes, node_to_bytes}; +#[pyfunction] +pub fn native_opcodes_dict(py: Python) -> PyResult { + let opcode_lookup: [(OpFn, &str); 30] = [ + (op_if, "op_if"), + (op_cons, "op_cons"), + (op_first, "op_first"), + (op_rest, "op_rest"), + (op_listp, "op_listp"), + (op_raise, "op_raise"), + (op_eq, "op_eq"), + (op_sha256, "op_sha256"), + (op_add, "op_add"), + (op_subtract, "op_subtract"), + (op_multiply, "op_multiply"), + (op_divmod, "op_divmod"), + (op_substr, "op_substr"), + (op_strlen, "op_strlen"), + (op_point_add, "op_point_add"), + (op_pubkey_for_exp, "op_pubkey_for_exp"), + (op_concat, "op_concat"), + (op_gr, "op_gr"), + (op_gr_bytes, "op_gr_bytes"), + (op_logand, "op_logand"), + (op_logior, "op_logior"), + (op_logxor, "op_logxor"), + (op_lognot, "op_lognot"), + (op_ash, "op_ash"), + (op_lsh, "op_lsh"), + (op_not, "op_not"), + (op_any, "op_any"), + (op_all, "op_all"), + (op_softfork, "op_softfork"), + (op_div, "op_div"), + ]; + let r = PyDict::new(py); + for (f, name) in opcode_lookup.iter() { + r.set_item(name, PyCell::new(py, NativeOp::new(*f))?)?; + } + Ok(r.to_object(py)) +} #[pyfunction] fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult { @@ -56,8 +103,6 @@ fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult { deserialize_from_bytes_for_allocator(py, blob, &arena) } -use crate::node::Node; - #[pyfunction] fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { let arena = PyArena::new(py)?.borrow(); @@ -84,19 +129,22 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; - m.add("STRICT_MODE", STRICT_MODE)?; m.add_class::()?; m.add_function(wrap_pyfunction!(native_opcodes_dict, m)?)?; m.add_function(wrap_pyfunction!(serialized_length, m)?)?; + m.add( + "NATIVE_OP_UNKNOWN_NON_STRICT", + PyMultiOpFn::new(|_a, _b, op, _d| err(op, "unimplemented operator")), + )?; + + m.add("NATIVE_OP_UNKNOWN_STRICT", PyMultiOpFn::new(op_unknown))?; + Ok(()) } -use crate::py::op_fn::PyOperatorHandler; -use crate::reduction::{EvalErr, Reduction}; - #[pyfunction] #[allow(clippy::too_many_arguments)] pub fn py_run_program<'p>( diff --git a/src/py/dialect.rs b/src/py/dialect.rs index b7aff7af..fbb68022 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -1,16 +1,14 @@ use std::cell::RefMut; use std::collections::HashMap; -use pyo3::prelude::{pyclass, pyfunction, pymethods}; +use pyo3::prelude::{pyclass, pymethods}; use pyo3::types::{PyBytes, PyDict, PyString, PyTuple}; use pyo3::{FromPyObject, PyCell, PyErr, PyObject, PyRef, PyResult, Python, ToPyObject}; use crate::allocator::Allocator; use crate::cost::Cost; -use crate::err_utils::err; use crate::int_allocator::IntAllocator; -use crate::more_ops::op_unknown; use crate::reduction::EvalErr; use crate::reduction::Reduction; use crate::reduction::Response; @@ -22,6 +20,18 @@ use super::f_table::OpFn; use super::native_op::NativeOp; use super::py_arena::PyArena; +#[pyclass] +#[derive(Clone)] +pub struct PyMultiOpFn { + op: MultiOpFn, +} + +impl PyMultiOpFn { + pub fn new(op: MultiOpFn) -> Self { + Self { op } + } +} + pub type MultiOpFn = fn( &mut T, ::AtomBuf, @@ -32,14 +42,7 @@ pub type MultiOpFn = fn( #[derive(Clone)] pub enum MultiOpFnE { Python(PyObject), - Rust( - fn( - &mut T, - ::AtomBuf, - ::Ptr, - Cost, - ) -> Response<::Ptr>, - ), + Rust(MultiOpFn), } impl MultiOpFnE { @@ -51,8 +54,8 @@ impl MultiOpFnE { max_cost: Cost, ) -> Response<::Ptr> { match self { - Self::Python(o) => { - panic!("oops") + Self::Python(_o) => { + todo!() } Self::Rust(f) => f(allocator, o, args, max_cost), } @@ -60,18 +63,10 @@ impl MultiOpFnE { } impl<'source> FromPyObject<'source> for MultiOpFnE { - fn extract(obj: &'source pyo3::PyAny) -> std::result::Result { - let v: PyResult = obj.extract(); + fn extract(obj: &'source pyo3::PyAny) -> PyResult { + let v: PyResult<&PyCell> = obj.extract(); if let Ok(v) = v { - Ok(Self::Rust(if v == 0 { - op_unknown - } else { - |_a, _b, op, _d| { - //let buf = op.to_vec(); - //let op_arg = allocator.new_atom(&buf)?; - err(op, "unimplemented operator") - } - })) + Ok(Self::Rust(v.borrow().op)) } else { Ok(Self::Python(obj.into())) } @@ -291,47 +286,3 @@ fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult Ok(ctx.into()), } } - -use crate::core_ops::*; -use crate::more_ops::*; - -#[pyfunction] -pub fn native_opcodes_dict(py: Python) -> PyResult { - let opcode_lookup: [(OpFn, &str); 30] = [ - (op_if, "op_if"), - (op_cons, "op_cons"), - (op_first, "op_first"), - (op_rest, "op_rest"), - (op_listp, "op_listp"), - (op_raise, "op_raise"), - (op_eq, "op_eq"), - (op_sha256, "op_sha256"), - (op_add, "op_add"), - (op_subtract, "op_subtract"), - (op_multiply, "op_multiply"), - (op_divmod, "op_divmod"), - (op_substr, "op_substr"), - (op_strlen, "op_strlen"), - (op_point_add, "op_point_add"), - (op_pubkey_for_exp, "op_pubkey_for_exp"), - (op_concat, "op_concat"), - (op_gr, "op_gr"), - (op_gr_bytes, "op_gr_bytes"), - (op_logand, "op_logand"), - (op_logior, "op_logior"), - (op_logxor, "op_logxor"), - (op_lognot, "op_lognot"), - (op_ash, "op_ash"), - (op_lsh, "op_lsh"), - (op_not, "op_not"), - (op_any, "op_any"), - (op_all, "op_all"), - (op_softfork, "op_softfork"), - (op_div, "op_div"), - ]; - let r = PyDict::new(py); - for (f, name) in opcode_lookup.iter() { - r.set_item(name, PyCell::new(py, NativeOp::new(*f))?)?; - } - Ok(r.to_object(py)) -} diff --git a/src/py/run_program.rs b/src/py/run_program.rs index a44c017b..7eebccdb 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -17,8 +17,6 @@ use super::py_arena::PyArena; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict}; -pub const STRICT_MODE: u32 = 1; - pub struct OperatorHandlerWithMode { f_lookup: FLookup, strict: bool, @@ -64,14 +62,16 @@ pub fn deserialize_and_run_program( let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; let f_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); - let strict: bool = (flags & STRICT_MODE) != 0; + let strict: bool = flags != 0; let f = OperatorHandlerWithMode { f_lookup, strict }; let program = node_from_bytes(allocator, program)?; let args = node_from_bytes(allocator, args)?; - let r = run_program( - allocator, &program, &args, quote_kw, apply_kw, max_cost, &f, None, - ); + let r = py.allow_threads(|| { + run_program( + allocator, &program, &args, quote_kw, apply_kw, max_cost, &f, None, + ) + }); match r { Ok(reduction) => Ok(( reduction.0, From 7e7a7bc93e8d33bf2fdfd604c0965308d6b09231 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 14 Apr 2021 17:08:47 -0700 Subject: [PATCH 45/76] Add api to `Arena`. --- src/py/api.rs | 6 +-- src/py/arena_object.rs | 8 +++- src/py/dialect.rs | 88 +++++++++++++++++++++++++++++++++++++----- src/py/py_arena.rs | 24 +++++++++++- src/py/run_program.rs | 2 +- 5 files changed, 112 insertions(+), 16 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index a4ea9c53..12388441 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -99,13 +99,13 @@ fn deserialize_from_bytes_for_allocator<'p>( #[pyfunction] fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult { - let arena = PyArena::new(py)?; + let arena = PyArena::new_cell(py)?; deserialize_from_bytes_for_allocator(py, blob, &arena) } #[pyfunction] fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { - let arena = PyArena::new(py)?.borrow(); + let arena = PyArena::new_cell(py)?.borrow(); let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; @@ -157,7 +157,7 @@ pub fn py_run_program<'p>( opcode_lookup_by_name: HashMap>, py_callback: PyObject, ) -> PyResult<(Cost, PyObject)> { - let arena = PyArena::new(py)?.borrow(); + let arena = PyArena::new_cell(py)?.borrow(); let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; diff --git a/src/py/arena_object.rs b/src/py/arena_object.rs index f1bb19dc..e364faff 100644 --- a/src/py/arena_object.rs +++ b/src/py/arena_object.rs @@ -35,7 +35,13 @@ impl ArenaObject { } #[getter(arena)] - fn arena<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { + pub fn get_arena<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { self.arena.extract(py) } } + +impl From<&ArenaObject> for i32 { + fn from(obj: &ArenaObject) -> Self { + obj.ptr + } +} diff --git a/src/py/dialect.rs b/src/py/dialect.rs index fbb68022..e30e5a23 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -13,7 +13,9 @@ use crate::reduction::EvalErr; use crate::reduction::Reduction; use crate::reduction::Response; use crate::run_program::OperatorHandler; +use crate::serialize::node_from_bytes; +use super::arena_object::ArenaObject; use super::clvm_object::CLVMObject; use super::f_table::FLookup; use super::f_table::OpFn; @@ -73,6 +75,12 @@ impl<'source> FromPyObject<'source> for MultiOpFnE { } } +fn same_arena(arena1: &PyArena, arena2: &PyArena) -> bool { + let p1: *const PyArena = arena1 as *const PyArena; + let p2: *const PyArena = arena2 as *const PyArena; + p1 == p2 +} + #[pyclass] pub struct Dialect { quote_kw: u8, @@ -126,18 +134,77 @@ impl Dialect { args: &PyCell, max_cost: Cost, ) -> PyResult<(Cost, PyObject)> { - let arena = PyArena::new(py)?; - let arena = &arena.borrow() as &PyArena; - let drc = DialectRunningContext { - dialect: self, - arena, + let arena = PyArena::new_cell(py)?; + + let (program, args) = { + let arena_ptr: &PyArena = &arena.borrow() as &PyArena; + let mut allocator_refcell: RefMut = arena_ptr.allocator(); + let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; + + let program = arena_ptr.native_for_py(py, program, allocator)?; + let args = arena_ptr.native_for_py(py, args, allocator)?; + (program, args) }; + let (cost, r) = self.run_program_ptr(py, &arena, program, args, max_cost)?; - let mut allocator_refcell: RefMut = arena.allocator(); + let arena_ptr: &PyArena = &arena.borrow() as &PyArena; + let mut allocator_refcell: RefMut = arena_ptr.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let program = arena.native_for_py(py, program, allocator)?; - let args = arena.native_for_py(py, args, allocator)?; + let r_ptr = &(&r).into(); + let new_r = arena_ptr.py_for_native(py, r_ptr, allocator)?; + Ok((cost, new_r.into())) + } + + pub fn run_program_arena<'p>( + &self, + py: Python<'p>, + program: &ArenaObject, + args: &ArenaObject, + max_cost: Cost, + ) -> PyResult<(Cost, ArenaObject)> { + let arena = program.get_arena(py)?; + if !same_arena(&arena.borrow(), &args.get_arena(py)?.borrow()) { + py.eval("raise ValueError('mismatched arenas')", None, None)?; + } + self.run_program_ptr(py, arena, program.into(), args.into(), max_cost) + } + + pub fn deserialize_and_run_program<'p>( + &self, + py: Python<'p>, + program_blob: &[u8], + args_blob: &[u8], + max_cost: Cost, + ) -> PyResult<(Cost, ArenaObject)> { + let arena = PyArena::new_cell(py)?; + let arena_ptr: &PyArena = &arena.borrow() as &PyArena; + let mut allocator_refcell: RefMut = arena_ptr.allocator(); + let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; + + let program = node_from_bytes(allocator, program_blob)?; + let args = node_from_bytes(allocator, args_blob)?; + self.run_program_ptr(py, &arena, program, args, max_cost) + } +} + +impl Dialect { + pub fn run_program_ptr<'p>( + &self, + py: Python<'p>, + arena: &PyCell, + program: i32, + args: i32, + max_cost: Cost, + ) -> PyResult<(Cost, ArenaObject)> { + let borrowed_arena = arena.borrow(); + let mut allocator_refcell: RefMut = borrowed_arena.allocator(); + let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; + + let drc = DialectRunningContext { + dialect: self, + arena: &borrowed_arena, + }; let r: Result, EvalErr> = crate::run_program::run_program( allocator, @@ -152,11 +219,12 @@ impl Dialect { match r { Ok(reduction) => { - let r = arena.py_for_native(py, &reduction.1, allocator)?; - Ok((reduction.0, r.to_object(py))) + let r = ArenaObject::new(py, arena, reduction.1); + Ok((reduction.0, r)) } Err(eval_err) => { let node: PyObject = arena + .borrow() .py_for_native(py, &eval_err.0, allocator)? .to_object(py); let s: String = eval_err.1; diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index 5c656918..7e9bfa72 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -6,7 +6,9 @@ use pyo3::types::{IntoPyDict, PyBytes, PyTuple}; use crate::allocator::{Allocator, SExp}; use crate::int_allocator::IntAllocator; +use crate::serialize::node_from_bytes; +use super::arena_object::ArenaObject; use super::clvm_object::CLVMObject; use super::py_view::PyView; @@ -16,8 +18,28 @@ pub struct PyArena { cache: PyObject, } +#[pymethods] impl PyArena { - pub fn new(py: Python) -> PyResult<&PyCell> { + #[new] + pub fn new(py: Python) -> PyResult { + Ok(PyArena { + arena: RefCell::new(IntAllocator::default()), + cache: py.eval("dict()", None, None)?.to_object(py), + }) + } + + pub fn deserialize(slf: &PyCell, py: Python, blob: &[u8]) -> PyResult { + let ptr = { + let borrowed_arena = slf.borrow(); + let allocator: &mut IntAllocator = &mut borrowed_arena.allocator() as &mut IntAllocator; + node_from_bytes(allocator, blob)? + }; + Ok(ArenaObject::new(py, slf, ptr)) + } +} + +impl PyArena { + pub fn new_cell(py: Python) -> PyResult<&PyCell> { PyCell::new( py, PyArena { diff --git a/src/py/run_program.rs b/src/py/run_program.rs index 7eebccdb..1383cd85 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -58,7 +58,7 @@ pub fn deserialize_and_run_program( max_cost: Cost, flags: u32, ) -> PyResult<(Cost, PyObject)> { - let arena = PyArena::new(py)?.borrow(); + let arena = PyArena::new(py)?; let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; let f_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); From 8f6417ee97b872e7b3431c452bf96fbb56a08f94 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 30 Apr 2021 16:44:55 -0700 Subject: [PATCH 46/76] Minor refactor. --- src/py/dialect.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/py/dialect.rs b/src/py/dialect.rs index e30e5a23..d5b6e0e7 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -178,12 +178,15 @@ impl Dialect { max_cost: Cost, ) -> PyResult<(Cost, ArenaObject)> { let arena = PyArena::new_cell(py)?; - let arena_ptr: &PyArena = &arena.borrow() as &PyArena; - let mut allocator_refcell: RefMut = arena_ptr.allocator(); - let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; + let (program, args) = { + let arena_ptr: &PyArena = &arena.borrow() as &PyArena; + let mut allocator_refcell: RefMut = arena_ptr.allocator(); + let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let program = node_from_bytes(allocator, program_blob)?; - let args = node_from_bytes(allocator, args_blob)?; + let program = node_from_bytes(allocator, program_blob)?; + let args = node_from_bytes(allocator, args_blob)?; + (program, args) + }; self.run_program_ptr(py, &arena, program, args, max_cost) } } From 296793fd574020cddfa78a22d7a0f067b51f1ab6 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 30 Apr 2021 16:45:28 -0700 Subject: [PATCH 47/76] Use `PyAny` instead of `PyCell`. --- src/py/arena_object.rs | 7 ++++--- src/py/py_arena.rs | 44 ++++++++++++++++++++++-------------------- src/py/py_view.rs | 1 + 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/py/arena_object.rs b/src/py/arena_object.rs index e364faff..15b13759 100644 --- a/src/py/arena_object.rs +++ b/src/py/arena_object.rs @@ -1,6 +1,7 @@ use std::cell::RefMut; use pyo3::prelude::{pyclass, pymethods}; +use pyo3::PyAny; use pyo3::PyCell; use pyo3::PyObject; use pyo3::PyRef; @@ -10,7 +11,6 @@ use pyo3::ToPyObject; use crate::int_allocator::IntAllocator; -use super::clvm_object::CLVMObject; use super::py_arena::PyArena; #[pyclass(weakref, subclass)] @@ -28,10 +28,11 @@ impl ArenaObject { #[pymethods] impl ArenaObject { - fn clvm_object<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { + fn clvm_object<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { let arena: PyRef = self.arena.extract(py)?; let mut allocator: RefMut = arena.allocator(); - arena.py_for_native(py, &self.ptr, &mut allocator) + arena + .py_for_native(py, &self.ptr, &mut allocator) } #[getter(arena)] diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index 7e9bfa72..be3bd644 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -38,6 +38,11 @@ impl PyArena { } } +fn py_view_for_obj(obj: &PyAny) -> PyResult { + let node: &PyCell = obj.extract()?; + Ok(node.borrow().py_view.clone()) +} + impl PyArena { pub fn new_cell(py: Python) -> PyResult<&PyCell> { PyCell::new( @@ -56,7 +61,7 @@ impl PyArena { pub fn add( &self, py: Python, - obj: &PyCell, + obj: &PyAny, ptr: &::Ptr, ) -> PyResult<()> { let locals = [ @@ -74,7 +79,7 @@ impl PyArena { fn from_py_to_native_cache<'p>( &self, py: Python<'p>, - obj: &PyCell, + obj: &PyAny, ) -> PyResult<::Ptr> { let locals = [("cache", self.cache.clone()), ("key", obj.to_object(py))].into_py_dict(py); py.eval("cache.get(id(key))", None, Some(locals))?.extract() @@ -83,41 +88,39 @@ impl PyArena { fn populate_native( &self, py: Python, - obj: &PyCell, + obj: &PyAny, allocator: &mut IntAllocator, ) -> PyResult<::Ptr> { - apply_to_tree(obj.to_object(py), move |obj| { - let node: &PyCell = obj.extract(py)?; - + apply_to_tree(obj, move |obj| { // is it in cache yet? - if self.from_py_to_native_cache(py, node).is_ok() { + if self.from_py_to_native_cache(py, obj).is_ok() { // yep, we're done return Ok(None); } // it's not in the cache - match &node.borrow().py_view { - PyView::Atom(obj) => { - let blob: &[u8] = obj.extract(py).unwrap(); + match py_view_for_obj(obj)? { + PyView::Atom(atom) => { + let blob: &[u8] = atom.extract(py).unwrap(); let ptr = allocator.new_atom(blob).unwrap(); - self.add(py, node, &ptr)?; + self.add(py, obj, &ptr)?; Ok(None) } PyView::Pair(pair) => { let pair: &PyAny = pair.clone().into_ref(py); let pair: &PyTuple = pair.extract()?; - let p0: &PyCell = pair.get_item(0).extract()?; - let p1: &PyCell = pair.get_item(1).extract()?; + let p0: &PyAny = pair.get_item(0); + let p1: &PyAny = pair.get_item(1); let ptr_0: PyResult = self.from_py_to_native_cache(py, p0); let ptr_1: PyResult = self.from_py_to_native_cache(py, p1); if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); - self.add(py, node, &ptr)?; + self.add(py, obj, &ptr)?; Ok(None) } else { - Ok(Some((p0.to_object(py), p1.to_object(py)))) + Ok(Some((p0, p1))) } } } @@ -129,7 +132,7 @@ impl PyArena { pub fn native_for_py( &self, py: Python, - obj: &PyCell, + obj: &PyAny, allocator: &mut IntAllocator, ) -> PyResult<::Ptr> { self.from_py_to_native_cache(py, obj) @@ -142,7 +145,7 @@ impl PyArena { &self, py: Python<'p>, ptr: &::Ptr, - ) -> PyResult<&'p PyCell> { + ) -> PyResult<&'p PyAny> { let locals = [("cache", self.cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); py.eval("cache[key]", None, Some(locals))?.extract() } @@ -152,7 +155,7 @@ impl PyArena { py: Python<'p>, ptr: &::Ptr, allocator: &mut IntAllocator, - ) -> PyResult<&'p PyCell> { + ) -> PyResult<&'p PyAny> { apply_to_tree(*ptr, move |ptr| { // is it in cache yet? if self.from_native_to_py_cache(py, &ptr).is_ok() { @@ -193,8 +196,7 @@ impl PyArena { // the children are in the cache, create new node & populate cache with it Ok(tuple) => { - let (p1, p2): (&PyCell, &PyCell) = - tuple.extract()?; + let (p1, p2): (&PyAny, &PyAny) = tuple.extract()?; self.add( py, CLVMObject::new( @@ -218,7 +220,7 @@ impl PyArena { py: Python<'p>, ptr: &::Ptr, allocator: &mut IntAllocator, - ) -> PyResult<&'p PyCell> { + ) -> PyResult<&'p PyAny> { self.from_native_to_py_cache(py, ptr) .or_else(|_err| self.populate_python(py, ptr, allocator)) } diff --git a/src/py/py_view.rs b/src/py/py_view.rs index 9c025b52..7e02c93e 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -4,6 +4,7 @@ use pyo3::{PyObject, PyResult, Python, ToPyObject}; use super::clvm_object::CLVMObject; +#[derive(Clone)] pub enum PyView { Atom(PyObject), Pair(PyObject), From 33d279e5762c0ecc1ca05be16e111fda9c34805f Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 4 May 2021 16:28:39 -0700 Subject: [PATCH 48/76] Deal with import loops. --- src/py/arena_object.rs | 18 ++++++++++++++++-- src/py/py_arena.rs | 33 ++++++++++++++++++++++++++++----- src/py/py_view.rs | 16 ++++++++++++++-- 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/src/py/arena_object.rs b/src/py/arena_object.rs index 15b13759..36026738 100644 --- a/src/py/arena_object.rs +++ b/src/py/arena_object.rs @@ -31,14 +31,28 @@ impl ArenaObject { fn clvm_object<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { let arena: PyRef = self.arena.extract(py)?; let mut allocator: RefMut = arena.allocator(); - arena - .py_for_native(py, &self.ptr, &mut allocator) + arena.py_for_native(py, &self.ptr, &mut allocator) } #[getter(arena)] pub fn get_arena<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { self.arena.extract(py) } + + #[getter(ptr)] + pub fn get_ptr(&self) -> i32 { + self.ptr + } + + #[getter(atom)] + pub fn get_atom<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { + self.clvm_object(py)?.getattr("atom") + } + + #[getter(pair)] + pub fn get_pair<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { + self.clvm_object(py)?.getattr("pair") + } } impl From<&ArenaObject> for i32 { diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index be3bd644..78207398 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -1,4 +1,5 @@ use std::cell::{RefCell, RefMut}; +use std::collections::HashSet; use pyo3::prelude::pyclass; use pyo3::prelude::*; @@ -36,11 +37,14 @@ impl PyArena { }; Ok(ArenaObject::new(py, slf, ptr)) } -} -fn py_view_for_obj(obj: &PyAny) -> PyResult { - let node: &PyCell = obj.extract()?; - Ok(node.borrow().py_view.clone()) + pub fn include(slf: &PyCell, py: Python, obj: &PyAny) -> PyResult { + let borrowed_arena = slf.borrow(); + let allocator: &mut IntAllocator = &mut borrowed_arena.allocator() as &mut IntAllocator; + + let ptr = borrowed_arena.populate_native(py, obj, allocator)?; + Ok(ArenaObject::new(py, slf, ptr)) + } } impl PyArena { @@ -91,6 +95,11 @@ impl PyArena { obj: &PyAny, allocator: &mut IntAllocator, ) -> PyResult<::Ptr> { + // items in `pending` are already in the stack of things to be converted + // if they appear again, we have an illegal cycle and must fail + + let mut pending: HashSet<*const PyObject> = HashSet::new(); + apply_to_tree(obj, move |obj| { // is it in cache yet? if self.from_py_to_native_cache(py, obj).is_ok() { @@ -100,7 +109,7 @@ impl PyArena { // it's not in the cache - match py_view_for_obj(obj)? { + match PyView::py_view_for_obj(obj)? { PyView::Atom(atom) => { let blob: &[u8] = atom.extract(py).unwrap(); let ptr = allocator.new_atom(blob).unwrap(); @@ -115,11 +124,25 @@ impl PyArena { let p1: &PyAny = pair.get_item(1); let ptr_0: PyResult = self.from_py_to_native_cache(py, p0); let ptr_1: PyResult = self.from_py_to_native_cache(py, p1); + + let as_obj: PyObject = obj.into(); + let as_obj = &as_obj as *const PyObject; + if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); self.add(py, obj, &ptr)?; + + pending.remove(&as_obj); + Ok(None) } else { + if pending.contains(&as_obj) { + py.run("raise ValueError('illegal clvm object loop')", None, None)?; + panic!(); + } + + pending.insert(as_obj); + Ok(Some((p0, p1))) } } diff --git a/src/py/py_view.rs b/src/py/py_view.rs index 7e02c93e..e369c6a9 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -1,6 +1,5 @@ -use pyo3::pycell::PyCell; use pyo3::types::{PyBytes, PyTuple}; -use pyo3::{PyObject, PyResult, Python, ToPyObject}; +use pyo3::{pycell::PyCell, PyAny, PyObject, PyResult, Python, ToPyObject}; use super::clvm_object::CLVMObject; @@ -24,4 +23,17 @@ impl PyView { Ok(PyView::Pair(pair.to_object(py))) } + + pub fn py_view_for_obj(obj: &PyAny) -> PyResult { + let node: PyResult<&PyCell> = obj.extract(); + if node.is_ok() { + return Ok(node?.borrow().py_view.clone()); + } + let r: &PyAny = obj.getattr("atom")?.extract()?; + if !r.is_none() { + return Ok(PyView::Atom(r.into())); + } + let r: &PyAny = obj.getattr("pair")?.extract()?; + Ok(PyView::Pair(r.into())) + } } From 8c234a0b3a9bae321747aa660cfff4b57a658b00 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 5 May 2021 17:09:04 -0700 Subject: [PATCH 49/76] Cast to `usize` instead of `*const PyObject`. --- src/py/dialect.rs | 12 +++--------- src/py/py_arena.rs | 48 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/src/py/dialect.rs b/src/py/dialect.rs index d5b6e0e7..ebcba937 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -135,19 +135,13 @@ impl Dialect { max_cost: Cost, ) -> PyResult<(Cost, PyObject)> { let arena = PyArena::new_cell(py)?; + let arena_ptr: &PyArena = &arena.borrow() as &PyArena; - let (program, args) = { - let arena_ptr: &PyArena = &arena.borrow() as &PyArena; - let mut allocator_refcell: RefMut = arena_ptr.allocator(); - let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; + let program = arena_ptr.ptr_for_obj(py, program)?; + let args = arena_ptr.ptr_for_obj(py, args)?; - let program = arena_ptr.native_for_py(py, program, allocator)?; - let args = arena_ptr.native_for_py(py, args, allocator)?; - (program, args) - }; let (cost, r) = self.run_program_ptr(py, &arena, program, args, max_cost)?; - let arena_ptr: &PyArena = &arena.borrow() as &PyArena; let mut allocator_refcell: RefMut = arena_ptr.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index 78207398..3e33485f 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -38,12 +38,33 @@ impl PyArena { Ok(ArenaObject::new(py, slf, ptr)) } - pub fn include(slf: &PyCell, py: Python, obj: &PyAny) -> PyResult { + pub fn include<'p>( + slf: &'p PyCell, + py: Python<'p>, + obj: &'p PyAny, + ) -> PyResult<&'p PyCell> { let borrowed_arena = slf.borrow(); - let allocator: &mut IntAllocator = &mut borrowed_arena.allocator() as &mut IntAllocator; + let ptr = borrowed_arena.ptr_for_obj(py, obj)?; + PyCell::new(py, ArenaObject::new(py, slf, ptr)) + } - let ptr = borrowed_arena.populate_native(py, obj, allocator)?; - Ok(ArenaObject::new(py, slf, ptr)) + pub fn ptr_for_obj<'p>(&'p self, py: Python<'p>, obj: &'p PyAny) -> PyResult { + let allocator: &mut IntAllocator = &mut self.allocator() as &mut IntAllocator; + + /* + let arena_obj: PyResult> = obj.extract(); + if let Ok(arena) = arena_obj { + let same_arena: bool = { + let o1: PyObject = self; + o1 == arena.borrow().get_arena(py)?.into() + }; + if same_arena { + return Ok(arena); + } + }; + */ + + self.populate_native(py, obj, allocator) } } @@ -98,7 +119,7 @@ impl PyArena { // items in `pending` are already in the stack of things to be converted // if they appear again, we have an illegal cycle and must fail - let mut pending: HashSet<*const PyObject> = HashSet::new(); + let mut pending: HashSet = HashSet::new(); apply_to_tree(obj, move |obj| { // is it in cache yet? @@ -125,22 +146,24 @@ impl PyArena { let ptr_0: PyResult = self.from_py_to_native_cache(py, p0); let ptr_1: PyResult = self.from_py_to_native_cache(py, p1); - let as_obj: PyObject = obj.into(); - let as_obj = &as_obj as *const PyObject; + let as_obj = id_for_pyany(py, obj)?; if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); self.add(py, obj, &ptr)?; pending.remove(&as_obj); - Ok(None) } else { if pending.contains(&as_obj) { - py.run("raise ValueError('illegal clvm object loop')", None, None)?; + let locals = Some([("obj", obj)].into_py_dict(py)); + py.run( + "raise ValueError(f'illegal clvm object loop {obj}')", + None, + locals, + )?; panic!(); } - pending.insert(as_obj); Ok(Some((p0, p1))) @@ -249,6 +272,11 @@ impl PyArena { } } +fn id_for_pyany(py: Python, obj: &PyAny) -> PyResult { + let locals = Some([("obj", obj)].into_py_dict(py)); + py.eval("id(obj)", None, locals)?.extract() +} + fn apply_to_tree(node: T, mut apply: F) -> PyResult<()> where F: FnMut(T) -> PyResult>, From 1bd36bf213512d0b7b648071ea38d8fd885b73de Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 5 May 2021 18:24:15 -0700 Subject: [PATCH 50/76] Get rid of most `CLVMObject` references. --- src/py/api.rs | 8 +++----- src/py/dialect.rs | 11 +++++------ src/py/op_fn.rs | 7 +++---- src/py/py_arena.rs | 2 +- src/py/py_view.rs | 12 +++--------- 5 files changed, 15 insertions(+), 25 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 12388441..609d1b5e 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -16,7 +16,6 @@ use crate::reduction::{EvalErr, Reduction}; use crate::serialize::{node_from_bytes, node_to_bytes}; use super::arena_object::ArenaObject; -use super::clvm_object::CLVMObject; use super::dialect::{Dialect, PyMultiOpFn}; use super::f_table::OpFn; use super::native_op::NativeOp; @@ -104,7 +103,7 @@ fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult { } #[pyfunction] -fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult<&'p PyBytes> { +fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyAny) -> PyResult<&'p PyBytes> { let arena = PyArena::new_cell(py)?.borrow(); let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; @@ -121,7 +120,6 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyCell) -> PyResult fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; - m.add_class::()?; m.add_function(wrap_pyfunction!(py_run_program, m)?)?; m.add_function(wrap_pyfunction!(deserialize_from_bytes, m)?)?; @@ -149,8 +147,8 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { #[allow(clippy::too_many_arguments)] pub fn py_run_program<'p>( py: Python<'p>, - program: &PyCell, - args: &PyCell, + program: &PyAny, + args: &PyAny, quote_kw: u8, apply_kw: u8, max_cost: Cost, diff --git a/src/py/dialect.rs b/src/py/dialect.rs index ebcba937..fd2d2d3a 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use pyo3::prelude::{pyclass, pymethods}; use pyo3::types::{PyBytes, PyDict, PyString, PyTuple}; -use pyo3::{FromPyObject, PyCell, PyErr, PyObject, PyRef, PyResult, Python, ToPyObject}; +use pyo3::{FromPyObject, PyAny, PyCell, PyErr, PyObject, PyRef, PyResult, Python, ToPyObject}; use crate::allocator::Allocator; use crate::cost::Cost; @@ -16,7 +16,6 @@ use crate::run_program::OperatorHandler; use crate::serialize::node_from_bytes; use super::arena_object::ArenaObject; -use super::clvm_object::CLVMObject; use super::f_table::FLookup; use super::f_table::OpFn; use super::native_op::NativeOp; @@ -130,8 +129,8 @@ impl Dialect { pub fn run_program<'p>( &self, py: Python<'p>, - program: &PyCell, - args: &PyCell, + program: &PyAny, + args: &PyAny, max_cost: Cost, ) -> PyResult<(Cost, PyObject)> { let arena = PyArena::new_cell(py)?; @@ -272,7 +271,7 @@ impl DialectRunningContext<'_> { let i0: u32 = unwrap_or_eval_err(pair.get_item(0).extract(), args, "expected u32")?; - let clvm_object: &PyCell = + let clvm_object: &PyAny = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; let r = self.arena.native_for_py(py, clvm_object, allocator); @@ -321,7 +320,7 @@ fn eval_err_for_pyerr<'p>( ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; - let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; + let sexp: &PyAny = pyerr.pvalue(py).getattr("_sexp")?.extract()?; let node: i32 = arena.native_for_py(py, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 2964fa69..5af6fea8 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use pyo3::types::PyBytes; use pyo3::types::PyString; use pyo3::types::PyTuple; -use pyo3::PyCell; +use pyo3::PyAny; use pyo3::PyErr; use pyo3::PyObject; use pyo3::PyResult; @@ -16,7 +16,6 @@ use crate::int_allocator::IntAllocator; use crate::reduction::{EvalErr, Reduction, Response}; use crate::run_program::OperatorHandler; -use super::clvm_object::CLVMObject; use super::f_table::{f_lookup_for_hashmap, FLookup}; use super::py_arena::PyArena; @@ -70,7 +69,7 @@ impl<'p> PyOperatorHandler<'p> { let i0: u32 = unwrap_or_eval_err(pair.get_item(0).extract(), args, "expected u32")?; - let clvm_object: &PyCell = + let clvm_object: &PyAny = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; let r = self.arena.native_for_py(py, clvm_object, allocator); @@ -111,7 +110,7 @@ fn eval_err_for_pyerr<'p>( ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; - let sexp: &PyCell = pyerr.pvalue(py).getattr("_sexp")?.extract()?; + let sexp: &PyAny = pyerr.pvalue(py).getattr("_sexp")?.extract()?; let node: i32 = arena.native_for_py(py, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index 3e33485f..3ae77af4 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -139,7 +139,7 @@ impl PyArena { Ok(None) } PyView::Pair(pair) => { - let pair: &PyAny = pair.clone().into_ref(py); + let pair: &PyAny = pair.into_ref(py); let pair: &PyTuple = pair.extract()?; let p0: &PyAny = pair.get_item(0); let p1: &PyAny = pair.get_item(1); diff --git a/src/py/py_view.rs b/src/py/py_view.rs index e369c6a9..9b9ee280 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -1,7 +1,5 @@ use pyo3::types::{PyBytes, PyTuple}; -use pyo3::{pycell::PyCell, PyAny, PyObject, PyResult, Python, ToPyObject}; - -use super::clvm_object::CLVMObject; +use pyo3::{PyAny, PyObject, PyResult, Python, ToPyObject}; #[derive(Clone)] pub enum PyView { @@ -18,17 +16,13 @@ impl PyView { if pair.len() != 2 { py.eval("raise ValueError('new_pair requires 2-tuple')", None, None)?; } - let _p0: &PyCell = pair.get_item(0).extract()?; - let _p1: &PyCell = pair.get_item(1).extract()?; + let _p0: &PyAny = pair.get_item(0).extract()?; + let _p1: &PyAny = pair.get_item(1).extract()?; Ok(PyView::Pair(pair.to_object(py))) } pub fn py_view_for_obj(obj: &PyAny) -> PyResult { - let node: PyResult<&PyCell> = obj.extract(); - if node.is_ok() { - return Ok(node?.borrow().py_view.clone()); - } let r: &PyAny = obj.getattr("atom")?.extract()?; if !r.is_none() { return Ok(PyView::Atom(r.into())); From c7b845c24edd0428271a231a7b07d04ef086a42d Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 10 May 2021 12:27:59 -0700 Subject: [PATCH 51/76] Use `&[u8]` instead of `u8` for apply & quote. --- src/py/api.rs | 4 ++-- src/py/dialect.rs | 12 ++++++------ src/py/run_program.rs | 4 ++-- src/run_program.rs | 16 ++++++++-------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 609d1b5e..c6901ac8 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -149,8 +149,8 @@ pub fn py_run_program<'p>( py: Python<'p>, program: &PyAny, args: &PyAny, - quote_kw: u8, - apply_kw: u8, + quote_kw: &[u8], + apply_kw: &[u8], max_cost: Cost, opcode_lookup_by_name: HashMap>, py_callback: PyObject, diff --git a/src/py/dialect.rs b/src/py/dialect.rs index fd2d2d3a..276dcf8f 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -82,8 +82,8 @@ fn same_arena(arena1: &PyArena, arena2: &PyArena) -> bool { #[pyclass] pub struct Dialect { - quote_kw: u8, - apply_kw: u8, + quote_kw: Vec, + apply_kw: Vec, u8_lookup: FLookup, python_u8_lookup: HashMap, PyObject>, native_u8_lookup: HashMap, OpFn>, @@ -95,8 +95,8 @@ impl Dialect { #[new] pub fn new( py: Python, - quote_kw: u8, - apply_kw: u8, + quote_kw: Vec, + apply_kw: Vec, op_table: HashMap, PyObject>, unknown_op_callback: MultiOpFnE, ) -> PyResult { @@ -206,8 +206,8 @@ impl Dialect { allocator, &program, &args, - self.quote_kw, - self.apply_kw, + &self.quote_kw, + &self.apply_kw, max_cost, &drc, None, diff --git a/src/py/run_program.rs b/src/py/run_program.rs index 1383cd85..b284b652 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -52,8 +52,8 @@ pub fn deserialize_and_run_program( py: Python, program: &[u8], args: &[u8], - quote_kw: u8, - apply_kw: u8, + quote_kw: &[u8], + apply_kw: &[u8], opcode_lookup_by_name: HashMap>, max_cost: Cost, flags: u32, diff --git a/src/run_program.rs b/src/run_program.rs index 4e6721d7..c7a5544c 100644 --- a/src/run_program.rs +++ b/src/run_program.rs @@ -51,8 +51,8 @@ enum Operation { pub struct RunProgramContext<'a, T: Allocator> { allocator: &'a mut T, - quote_kw: u8, - apply_kw: u8, + quote_kw: &'a [u8], + apply_kw: &'a [u8], operator_lookup: &'a dyn OperatorHandler, pre_eval: Option>, posteval_stack: Vec>>, @@ -159,8 +159,8 @@ fn augment_cost_errors( impl<'a, 'h, T: Allocator> RunProgramContext<'a, T> { fn new( allocator: &'a mut T, - quote_kw: u8, - apply_kw: u8, + quote_kw: &'a [u8], + apply_kw: &'a [u8], operator_lookup: &'a dyn OperatorHandler, pre_eval: Option>, ) -> Self { @@ -205,7 +205,7 @@ impl<'a, T: Allocator> RunProgramContext<'a, T> { ) -> Result> { let op_atom = self.allocator.buf(op_buf); // special case check for quote - if op_atom.len() == 1 && op_atom[0] == self.quote_kw { + if op_atom.len() == 1 && op_atom == self.quote_kw { self.push(operand_list.clone()); Ok(QUOTE_COST) } else { @@ -300,7 +300,7 @@ impl<'a, T: Allocator> RunProgramContext<'a, T> { SExp::Atom(opa) => opa, }; let op_atom = self.allocator.buf(&opa); - if op_atom.len() == 1 && op_atom[0] == self.apply_kw { + if op_atom.len() == 1 && op_atom == self.apply_kw { let operand_list = Node::new(self.allocator, operand_list); if operand_list.arg_count_is(2) { let new_operator = operand_list.first()?; @@ -374,8 +374,8 @@ pub fn run_program( allocator: &mut T, program: &T::Ptr, args: &T::Ptr, - quote_kw: u8, - apply_kw: u8, + quote_kw: &[u8], + apply_kw: &[u8], max_cost: Cost, operator_lookup: &dyn OperatorHandler, pre_eval: Option>, From 9fecb12302b1d521619e863f7dcf1be40631fc95 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 10 May 2021 13:37:43 -0700 Subject: [PATCH 52/76] Remove obsolete API. --- src/py/api.rs | 37 +++++++------------------------------ 1 file changed, 7 insertions(+), 30 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index c6901ac8..6e240799 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -13,7 +13,7 @@ use crate::int_allocator::IntAllocator; use crate::more_ops::*; use crate::node::Node; use crate::reduction::{EvalErr, Reduction}; -use crate::serialize::{node_from_bytes, node_to_bytes}; +use crate::serialize::node_to_bytes; use super::arena_object::ArenaObject; use super::dialect::{Dialect, PyMultiOpFn}; @@ -82,26 +82,6 @@ fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult( - py: Python<'p>, - blob: &[u8], - arena: &PyCell, -) -> PyResult { - let ptr = { - let arena: PyRef = arena.borrow(); - let allocator: &mut IntAllocator = &mut arena.allocator() as &mut IntAllocator; - node_from_bytes(allocator, blob)? - }; - Ok(ArenaObject::new(py, arena, ptr)) -} - -#[pyfunction] -fn deserialize_from_bytes(py: Python, blob: &[u8]) -> PyResult { - let arena = PyArena::new_cell(py)?; - deserialize_from_bytes_for_allocator(py, blob, &arena) -} - #[pyfunction] fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyAny) -> PyResult<&'p PyBytes> { let arena = PyArena::new_cell(py)?.borrow(); @@ -118,17 +98,10 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyAny) -> PyResult<&'p PyBytes> /// This module is a python module implemented in Rust. #[pymodule] fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; m.add_class::()?; - - m.add_function(wrap_pyfunction!(py_run_program, m)?)?; - m.add_function(wrap_pyfunction!(deserialize_from_bytes, m)?)?; - m.add_function(wrap_pyfunction!(deserialize_from_bytes_for_allocator, m)?)?; - m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; - - m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; - m.add_class::()?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(native_opcodes_dict, m)?)?; m.add_function(wrap_pyfunction!(serialized_length, m)?)?; @@ -140,6 +113,10 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add("NATIVE_OP_UNKNOWN_STRICT", PyMultiOpFn::new(op_unknown))?; + m.add_function(wrap_pyfunction!(py_run_program, m)?)?; + m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; + m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; + Ok(()) } From b81346da3177091663e157299264ff448d982665 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 10 May 2021 17:03:27 -0700 Subject: [PATCH 53/76] Add `error_bridge` and bridge to allowing calling native `OpFn` from py. --- src/py/dialect.rs | 52 +++++++----------------------------------- src/py/error_bridge.rs | 47 ++++++++++++++++++++++++++++++++++++++ src/py/mod.rs | 1 + src/py/native_op.rs | 40 +++++++++++++++++++++++++++++++- 4 files changed, 95 insertions(+), 45 deletions(-) create mode 100644 src/py/error_bridge.rs diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 276dcf8f..c7943d53 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -16,6 +16,7 @@ use crate::run_program::OperatorHandler; use crate::serialize::node_from_bytes; use super::arena_object::ArenaObject; +use super::error_bridge::{eval_err_for_pyerr, raise_eval_error, unwrap_or_eval_err}; use super::f_table::FLookup; use super::f_table::OpFn; use super::native_op::NativeOp; @@ -132,6 +133,7 @@ impl Dialect { program: &PyAny, args: &PyAny, max_cost: Cost, + pre_eval_f: &PyAny, ) -> PyResult<(Cost, PyObject)> { let arena = PyArena::new_cell(py)?; let arena_ptr: &PyArena = &arena.borrow() as &PyArena; @@ -139,7 +141,7 @@ impl Dialect { let program = arena_ptr.ptr_for_obj(py, program)?; let args = arena_ptr.ptr_for_obj(py, args)?; - let (cost, r) = self.run_program_ptr(py, &arena, program, args, max_cost)?; + let (cost, r) = self.run_program_ptr(py, &arena, program, args, max_cost, pre_eval_f)?; let mut allocator_refcell: RefMut = arena_ptr.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; @@ -155,12 +157,13 @@ impl Dialect { program: &ArenaObject, args: &ArenaObject, max_cost: Cost, + pre_eval: &PyAny, ) -> PyResult<(Cost, ArenaObject)> { let arena = program.get_arena(py)?; if !same_arena(&arena.borrow(), &args.get_arena(py)?.borrow()) { py.eval("raise ValueError('mismatched arenas')", None, None)?; } - self.run_program_ptr(py, arena, program.into(), args.into(), max_cost) + self.run_program_ptr(py, arena, program.into(), args.into(), max_cost, pre_eval) } pub fn deserialize_and_run_program<'p>( @@ -169,6 +172,7 @@ impl Dialect { program_blob: &[u8], args_blob: &[u8], max_cost: Cost, + pre_eval: &PyAny, ) -> PyResult<(Cost, ArenaObject)> { let arena = PyArena::new_cell(py)?; let (program, args) = { @@ -180,7 +184,7 @@ impl Dialect { let args = node_from_bytes(allocator, args_blob)?; (program, args) }; - self.run_program_ptr(py, &arena, program, args, max_cost) + self.run_program_ptr(py, &arena, program, args, max_cost, pre_eval) } } @@ -192,6 +196,7 @@ impl Dialect { program: i32, args: i32, max_cost: Cost, + _pre_eval: &PyAny, ) -> PyResult<(Cost, ArenaObject)> { let borrowed_arena = arena.borrow(); let mut allocator_refcell: RefMut = borrowed_arena.allocator(); @@ -309,44 +314,3 @@ impl OperatorHandler for DialectRunningContext<'_> { } } } - -/// turn a `PyErr` into an `EvalErr

` if at all possible -/// otherwise, return a `PyErr` -fn eval_err_for_pyerr<'p>( - py: Python<'p>, - pyerr: &PyErr, - arena: &'p PyArena, - allocator: &mut IntAllocator, -) -> PyResult> { - let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; - let arg0: &PyString = args.get_item(0).extract()?; - let sexp: &PyAny = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = arena.native_for_py(py, sexp, allocator)?; - let s: String = arg0.to_str()?.to_string(); - Ok(EvalErr(node, s)) -} - -fn unwrap_or_eval_err(obj: PyResult, err_node: &P, msg: &str) -> Result> -where - P: Clone, -{ - match obj { - Err(_py_err) => Err(EvalErr(err_node.clone(), msg.to_string())), - Ok(o) => Ok(o), - } -} - -fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult { - let ctx: &PyDict = PyDict::new(py); - ctx.set_item("msg", msg)?; - ctx.set_item("sexp", sexp)?; - let r = py.run( - "from clvm.EvalError import EvalError; raise EvalError(msg, sexp)", - None, - Some(ctx), - ); - match r { - Err(x) => Err(x), - Ok(_) => Ok(ctx.into()), - } -} diff --git a/src/py/error_bridge.rs b/src/py/error_bridge.rs new file mode 100644 index 00000000..9afd06cc --- /dev/null +++ b/src/py/error_bridge.rs @@ -0,0 +1,47 @@ +use crate::int_allocator::IntAllocator; +use crate::reduction::EvalErr; +use pyo3::types::{PyDict, PyString, PyTuple}; +use pyo3::{PyAny, PyErr, PyObject, PyResult, Python}; + +use super::py_arena::PyArena; + +/// turn a `PyErr` into an `EvalErr

` if at all possible +/// otherwise, return a `PyErr` +pub fn eval_err_for_pyerr<'p>( + py: Python<'p>, + pyerr: &PyErr, + arena: &'p PyArena, + allocator: &mut IntAllocator, +) -> PyResult> { + let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; + let arg0: &PyString = args.get_item(0).extract()?; + let sexp: &PyAny = pyerr.pvalue(py).getattr("_sexp")?.extract()?; + let node: i32 = arena.native_for_py(py, sexp, allocator)?; + let s: String = arg0.to_str()?.to_string(); + Ok(EvalErr(node, s)) +} + +pub fn unwrap_or_eval_err(obj: PyResult, err_node: &P, msg: &str) -> Result> +where + P: Clone, +{ + match obj { + Err(_py_err) => Err(EvalErr(err_node.clone(), msg.to_string())), + Ok(o) => Ok(o), + } +} + +pub fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult { + let ctx: &PyDict = PyDict::new(py); + ctx.set_item("msg", msg)?; + ctx.set_item("sexp", sexp)?; + let r = py.run( + "from clvm.EvalError import EvalError; raise EvalError(msg, sexp)", + None, + Some(ctx), + ); + match r { + Err(x) => Err(x), + Ok(_) => Ok(ctx.into()), + } +} diff --git a/src/py/mod.rs b/src/py/mod.rs index a3094bc7..7c3a5c40 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -2,6 +2,7 @@ pub mod api; pub mod arena_object; pub mod clvm_object; pub mod dialect; +pub mod error_bridge; pub mod f_table; pub mod native_op; pub mod op_fn; diff --git a/src/py/native_op.rs b/src/py/native_op.rs index 3d8a0adc..90b5135b 100644 --- a/src/py/native_op.rs +++ b/src/py/native_op.rs @@ -1,8 +1,16 @@ -use pyo3::prelude::pyclass; +use pyo3::prelude::{pyclass, pymethods}; +use pyo3::types::PyString; +use pyo3::{PyAny, PyCell, PyResult, Python, ToPyObject}; +use crate::allocator::Allocator; +use crate::cost::Cost; use crate::int_allocator::IntAllocator; +use crate::reduction::Reduction; +use super::arena_object::ArenaObject; +use super::error_bridge::{eval_err_for_pyerr, raise_eval_error}; use super::f_table::OpFn; +use super::py_arena::PyArena; //type OpFn = fn(&mut T, ::Ptr, Cost) -> Response<::Ptr>; @@ -16,3 +24,33 @@ impl NativeOp { Self { op } } } + +#[pymethods] +impl NativeOp { + #[call] + fn __call__(&self, py: Python, args: &PyAny, _max_cost: Cost) -> PyResult<(Cost, ArenaObject)> { + let arena_cell = PyArena::new_cell(py)?; + let ptr = PyArena::include(arena_cell, py, args)?.borrow().get_ptr(); + let arena: &PyArena = &arena_cell.borrow(); + let mut allocator = arena.allocator(); + let allocator: &mut IntAllocator = &mut allocator; + let r = (self.op)(allocator, ptr, _max_cost); + match r { + Ok(Reduction(cost, ptr)) => { + let r = ArenaObject::new(py, arena_cell, ptr); + Ok((cost, r)) + } + Err(err) => { + let r = ArenaObject::new(py, arena_cell, ptr); + match raise_eval_error( + py, + PyString::new(py, "problem in suboperator"), + PyCell::new(py, r)?.to_object(py), + ) { + Err(e) => Err(e), + Ok(_) => panic!("oh dear"), + } + } + } + } +} From 78c376aa2481cc8897cd02ce666acd690e20bd57 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 10 May 2021 17:27:33 -0700 Subject: [PATCH 54/76] Clippy --- src/py/dialect.rs | 4 ++-- src/py/native_op.rs | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/py/dialect.rs b/src/py/dialect.rs index c7943d53..74fa96c1 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -3,8 +3,8 @@ use std::collections::HashMap; use pyo3::prelude::{pyclass, pymethods}; -use pyo3::types::{PyBytes, PyDict, PyString, PyTuple}; -use pyo3::{FromPyObject, PyAny, PyCell, PyErr, PyObject, PyRef, PyResult, Python, ToPyObject}; +use pyo3::types::{PyBytes, PyString, PyTuple}; +use pyo3::{FromPyObject, PyAny, PyCell, PyObject, PyRef, PyResult, Python, ToPyObject}; use crate::allocator::Allocator; use crate::cost::Cost; diff --git a/src/py/native_op.rs b/src/py/native_op.rs index 90b5135b..333ea2f5 100644 --- a/src/py/native_op.rs +++ b/src/py/native_op.rs @@ -2,13 +2,12 @@ use pyo3::prelude::{pyclass, pymethods}; use pyo3::types::PyString; use pyo3::{PyAny, PyCell, PyResult, Python, ToPyObject}; -use crate::allocator::Allocator; use crate::cost::Cost; use crate::int_allocator::IntAllocator; use crate::reduction::Reduction; use super::arena_object::ArenaObject; -use super::error_bridge::{eval_err_for_pyerr, raise_eval_error}; +use super::error_bridge::raise_eval_error; use super::f_table::OpFn; use super::py_arena::PyArena; @@ -40,7 +39,7 @@ impl NativeOp { let r = ArenaObject::new(py, arena_cell, ptr); Ok((cost, r)) } - Err(err) => { + Err(_err) => { let r = ArenaObject::new(py, arena_cell, ptr); match raise_eval_error( py, From ae2f55628363bf5530efaf73b1b688affccde94e Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 11 May 2021 16:46:20 -0700 Subject: [PATCH 55/76] Reverse `NATIVE_OP_UNKNOWN_STRICT` and counterpart. --- src/py/api.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 6e240799..77de935a 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -107,11 +107,11 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(serialized_length, m)?)?; m.add( - "NATIVE_OP_UNKNOWN_NON_STRICT", + "NATIVE_OP_UNKNOWN_STRICT", PyMultiOpFn::new(|_a, _b, op, _d| err(op, "unimplemented operator")), )?; - m.add("NATIVE_OP_UNKNOWN_STRICT", PyMultiOpFn::new(op_unknown))?; + m.add("NATIVE_OP_UNKNOWN_NON_STRICT", PyMultiOpFn::new(op_unknown))?; m.add_function(wrap_pyfunction!(py_run_program, m)?)?; m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; From 5576cf14d08842c0fff4305644f9c5440a2c4c1d Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 11 May 2021 16:46:53 -0700 Subject: [PATCH 56/76] Add `Dialect.update`. --- src/py/dialect.rs | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 74fa96c1..88766882 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -95,36 +95,38 @@ pub struct Dialect { impl Dialect { #[new] pub fn new( - py: Python, quote_kw: Vec, apply_kw: Vec, - op_table: HashMap, PyObject>, unknown_op_callback: MultiOpFnE, ) -> PyResult { - let mut u8_lookup = [None; 256]; - let mut python_u8_lookup = HashMap::new(); - let mut native_u8_lookup = HashMap::new(); - for (op, fn_obj) in op_table.iter() { + let u8_lookup = [None; 256]; + let python_u8_lookup = HashMap::new(); + let native_u8_lookup = HashMap::new(); + Ok(Self { + quote_kw, + apply_kw, + u8_lookup, + python_u8_lookup, + native_u8_lookup, + unknown_op_callback, + }) + } + + pub fn update(&mut self, py: Python, d: HashMap, PyObject>) -> PyResult<()> { + for (op, fn_obj) in d.iter() { let r: PyResult> = fn_obj.extract(py); if let Ok(native_op) = r { if op.len() == 1 { let index = op[0] as usize; - u8_lookup[index] = Some(native_op.op); + self.u8_lookup[index] = Some(native_op.op); } else { - native_u8_lookup.insert(op.to_owned(), native_op.op); + self.native_u8_lookup.insert(op.to_owned(), native_op.op); } } else { - python_u8_lookup.insert(op.to_owned(), fn_obj.clone()); + self.python_u8_lookup.insert(op.to_owned(), fn_obj.clone()); } } - Ok(Self { - quote_kw, - apply_kw, - u8_lookup, - python_u8_lookup, - native_u8_lookup, - unknown_op_callback, - }) + Ok(()) } pub fn run_program<'p>( From dabce63fa41cee697c85b2d3def4c4242a8c9f5d Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 11 May 2021 16:47:32 -0700 Subject: [PATCH 57/76] Abstract out last `CLVMObject` reference. --- src/py/py_arena.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index 3ae77af4..4acbb6ee 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -17,6 +17,7 @@ use super::py_view::PyView; pub struct PyArena { arena: RefCell, cache: PyObject, + bridge_constructor: Box PyResult<&PyAny>>, } #[pymethods] @@ -26,6 +27,7 @@ impl PyArena { Ok(PyArena { arena: RefCell::new(IntAllocator::default()), cache: py.eval("dict()", None, None)?.to_object(py), + bridge_constructor: Box::new(|py, py_view| Ok(CLVMObject::new(py, py_view)?)), }) } @@ -70,13 +72,7 @@ impl PyArena { impl PyArena { pub fn new_cell(py: Python) -> PyResult<&PyCell> { - PyCell::new( - py, - PyArena { - arena: RefCell::new(IntAllocator::default()), - cache: py.eval("dict()", None, None)?.to_object(py), - }, - ) + PyCell::new(py, PyArena::new(py)?) } pub fn allocator(&self) -> RefMut { @@ -218,7 +214,7 @@ impl PyArena { let py_bytes = PyBytes::new(py, blob); self.add( py, - CLVMObject::new(py, PyView::new_atom(py, py_bytes))?, + (self.bridge_constructor)(py, PyView::new_atom(py, py_bytes))?, &ptr, )?; Ok(None) @@ -245,7 +241,7 @@ impl PyArena { let (p1, p2): (&PyAny, &PyAny) = tuple.extract()?; self.add( py, - CLVMObject::new( + (self.bridge_constructor)( py, PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?, )?, From a8e7b66705c3e76aefd321d2d8fe1a007b30b0e3 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 11 May 2021 17:35:55 -0700 Subject: [PATCH 58/76] Finish generalizing `bridge_constructor`. --- src/py/py_arena.rs | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index 4acbb6ee..07b29052 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -17,7 +17,17 @@ use super::py_view::PyView; pub struct PyArena { arena: RefCell, cache: PyObject, - bridge_constructor: Box PyResult<&PyAny>>, + bridge_constructor: Box Fn(Python<'p>, &'p PyAny) -> PyResult<&'p PyAny>>, +} + +fn clvm_obj_bridge<'p>(py: Python<'p>, obj: &'p PyAny) -> PyResult<&'p PyAny> { + if let Ok(b) = obj.extract::<&PyBytes>() { + Ok(CLVMObject::new(py, PyView::new_atom(py, b))?) + } else if let Ok(p) = obj.extract::<&PyTuple>() { + Ok(CLVMObject::new(py, PyView::new_pair(py, p)?)?) + } else { + panic!("bad bridge") + } } #[pymethods] @@ -27,7 +37,7 @@ impl PyArena { Ok(PyArena { arena: RefCell::new(IntAllocator::default()), cache: py.eval("dict()", None, None)?.to_object(py), - bridge_constructor: Box::new(|py, py_view| Ok(CLVMObject::new(py, py_view)?)), + bridge_constructor: Box::new(clvm_obj_bridge), }) } @@ -212,16 +222,12 @@ impl PyArena { // it's an atom, so we just populate cache directly let blob = allocator.buf(&a); let py_bytes = PyBytes::new(py, blob); - self.add( - py, - (self.bridge_constructor)(py, PyView::new_atom(py, py_bytes))?, - &ptr, - )?; + self.add(py, (self.bridge_constructor)(py, py_bytes)?, &ptr)?; Ok(None) } SExp::Pair(ptr_1, ptr_2) => { // we can only create this if the children are in the cache - // Let's fine out + // Let's find out let locals = [ ("cache", self.cache.clone()), ("p1", ptr_1.to_object(py)), @@ -238,15 +244,8 @@ impl PyArena { // the children are in the cache, create new node & populate cache with it Ok(tuple) => { - let (p1, p2): (&PyAny, &PyAny) = tuple.extract()?; - self.add( - py, - (self.bridge_constructor)( - py, - PyView::new_pair(py, PyTuple::new(py, &[p1, p2]))?, - )?, - &ptr, - )?; + let (_p1, _p2): (&PyAny, &PyAny) = tuple.extract()?; + self.add(py, (self.bridge_constructor)(py, tuple)?, &ptr)?; Ok(None) } } From 12721bfb7e1ec933d3d5fad3d8a99e72c2bd7187 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 11 May 2021 19:04:20 -0700 Subject: [PATCH 59/76] Need `&PyCell` in `native_for_py`. --- src/py/api.rs | 21 +++++++++++-------- src/py/dialect.rs | 16 +++++++------- src/py/error_bridge.rs | 6 +++--- src/py/op_fn.rs | 39 ++++++----------------------------- src/py/py_arena.rs | 47 ++++++++++++++++-------------------------- 5 files changed, 47 insertions(+), 82 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 77de935a..570d688f 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -84,11 +84,12 @@ fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult(py: Python<'p>, sexp: &PyAny) -> PyResult<&'p PyBytes> { - let arena = PyArena::new_cell(py)?.borrow(); - let mut allocator_refcell: RefMut = arena.allocator(); + let arena = PyArena::new_cell(py)?; + let arena_borrowed = arena.borrow(); + let mut allocator_refcell: RefMut = arena_borrowed.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let ptr = arena.native_for_py(py, sexp, allocator)?; + let ptr = PyArena::native_for_py(arena, py, sexp, allocator)?; let node = Node::new(allocator, ptr); let s: Vec = node_to_bytes(&node)?; @@ -132,13 +133,14 @@ pub fn py_run_program<'p>( opcode_lookup_by_name: HashMap>, py_callback: PyObject, ) -> PyResult<(Cost, PyObject)> { - let arena = PyArena::new_cell(py)?.borrow(); - let mut allocator_refcell: RefMut = arena.allocator(); + let arena = PyArena::new_cell(py)?; + let arena_borrowed = arena.borrow(); + let mut allocator_refcell: RefMut = arena_borrowed.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let op_lookup = PyOperatorHandler::new(opcode_lookup_by_name, py_callback, &arena)?; - let program = arena.native_for_py(py, program, allocator)?; - let args = arena.native_for_py(py, args, allocator)?; + let op_lookup = PyOperatorHandler::new(opcode_lookup_by_name, py_callback, arena)?; + let program = PyArena::native_for_py(arena, py, program, allocator)?; + let args = PyArena::native_for_py(arena, py, args, allocator)?; let r: Result, EvalErr> = crate::run_program::run_program( allocator, &program, &args, quote_kw, apply_kw, max_cost, &op_lookup, None, @@ -146,11 +148,12 @@ pub fn py_run_program<'p>( match r { Ok(reduction) => { - let r = arena.py_for_native(py, &reduction.1, allocator)?; + let r = arena.borrow().py_for_native(py, &reduction.1, allocator)?; Ok((reduction.0, r.to_object(py))) } Err(eval_err) => { let node: PyObject = arena + .borrow() .py_for_native(py, &eval_err.0, allocator)? .to_object(py); let s: String = eval_err.1; diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 88766882..da34b6e5 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -140,8 +140,8 @@ impl Dialect { let arena = PyArena::new_cell(py)?; let arena_ptr: &PyArena = &arena.borrow() as &PyArena; - let program = arena_ptr.ptr_for_obj(py, program)?; - let args = arena_ptr.ptr_for_obj(py, args)?; + let program = PyArena::ptr_for_obj(arena, py, program)?; + let args = PyArena::ptr_for_obj(arena, py, args)?; let (cost, r) = self.run_program_ptr(py, &arena, program, args, max_cost, pre_eval_f)?; @@ -206,7 +206,7 @@ impl Dialect { let drc = DialectRunningContext { dialect: self, - arena: &borrowed_arena, + arena: &arena, }; let r: Result, EvalErr> = crate::run_program::run_program( @@ -244,7 +244,7 @@ impl Dialect { struct DialectRunningContext<'a> { dialect: &'a Dialect, - arena: &'a PyArena, + arena: &'a PyCell, } impl DialectRunningContext<'_> { @@ -259,17 +259,17 @@ impl DialectRunningContext<'_> { Python::with_gil(|py| { let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); let r = unwrap_or_eval_err( - self.arena.py_for_native(py, args, allocator), + PyArena::py_for_native(&self.arena.borrow(), py, args, allocator), args, "can't uncache", )?; - let r1 = obj.call1(py, (op, r.to_object(py), max_cost)); + let r1 = obj.call1(py, (r.to_object(py), max_cost)); match r1 { Err(pyerr) => { let eval_err: PyResult> = eval_err_for_pyerr(py, &pyerr, self.arena, allocator); let r: EvalErr = - unwrap_or_eval_err(eval_err, args, "unexpected exception")?; + unwrap_or_eval_err(eval_err, args, "2unexpected exception")?; Err(r) } Ok(o) => { @@ -281,7 +281,7 @@ impl DialectRunningContext<'_> { let clvm_object: &PyAny = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = self.arena.native_for_py(py, clvm_object, allocator); + let r = PyArena::native_for_py(self.arena, py, clvm_object, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } diff --git a/src/py/error_bridge.rs b/src/py/error_bridge.rs index 9afd06cc..dd233d95 100644 --- a/src/py/error_bridge.rs +++ b/src/py/error_bridge.rs @@ -1,7 +1,7 @@ use crate::int_allocator::IntAllocator; use crate::reduction::EvalErr; use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{PyAny, PyErr, PyObject, PyResult, Python}; +use pyo3::{PyAny, PyCell, PyErr, PyObject, PyResult, Python}; use super::py_arena::PyArena; @@ -10,13 +10,13 @@ use super::py_arena::PyArena; pub fn eval_err_for_pyerr<'p>( py: Python<'p>, pyerr: &PyErr, - arena: &'p PyArena, + arena: &'p PyCell, allocator: &mut IntAllocator, ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; let sexp: &PyAny = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = arena.native_for_py(py, sexp, allocator)?; + let node: i32 = PyArena::native_for_py(arena, py, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) } diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs index 5af6fea8..f4cdd864 100644 --- a/src/py/op_fn.rs +++ b/src/py/op_fn.rs @@ -1,14 +1,12 @@ use std::collections::HashMap; use pyo3::types::PyBytes; -use pyo3::types::PyString; use pyo3::types::PyTuple; -use pyo3::PyAny; -use pyo3::PyErr; use pyo3::PyObject; use pyo3::PyResult; use pyo3::Python; use pyo3::ToPyObject; +use pyo3::{PyAny, PyCell}; use crate::allocator::Allocator; use crate::cost::Cost; @@ -16,20 +14,21 @@ use crate::int_allocator::IntAllocator; use crate::reduction::{EvalErr, Reduction, Response}; use crate::run_program::OperatorHandler; +use super::error_bridge::{eval_err_for_pyerr, unwrap_or_eval_err}; use super::f_table::{f_lookup_for_hashmap, FLookup}; use super::py_arena::PyArena; pub struct PyOperatorHandler<'p> { native_lookup: FLookup, py_callable: PyObject, - arena: &'p PyArena, + arena: &'p PyCell, } impl<'p> PyOperatorHandler<'p> { pub fn new( opcode_lookup_by_name: HashMap>, py_callable: PyObject, - arena: &'p PyArena, + arena: &'p PyCell, ) -> PyResult { let native_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); Ok(PyOperatorHandler { @@ -50,7 +49,7 @@ impl<'p> PyOperatorHandler<'p> { Python::with_gil(|py| { let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); let r = unwrap_or_eval_err( - self.arena.py_for_native(py, args, allocator), + PyArena::py_for_native(&self.arena.borrow(), py, args, allocator), args, "can't uncache", )?; @@ -72,7 +71,7 @@ impl<'p> PyOperatorHandler<'p> { let clvm_object: &PyAny = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = self.arena.native_for_py(py, clvm_object, allocator); + let r = PyArena::native_for_py(self.arena, py, clvm_object, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } @@ -99,29 +98,3 @@ impl OperatorHandler for PyOperatorHandler<'_> { self.invoke_py_obj(self.py_callable.clone(), allocator, op_buf, args, max_cost) } } - -/// turn a `PyErr` into an `EvalErr

` if at all possible -/// otherwise, return a `PyErr` -fn eval_err_for_pyerr<'p>( - py: Python<'p>, - pyerr: &PyErr, - arena: &'p PyArena, - allocator: &mut IntAllocator, -) -> PyResult> { - let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; - let arg0: &PyString = args.get_item(0).extract()?; - let sexp: &PyAny = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = arena.native_for_py(py, sexp, allocator)?; - let s: String = arg0.to_str()?.to_string(); - Ok(EvalErr(node, s)) -} - -fn unwrap_or_eval_err(obj: PyResult, err_node: &P, msg: &str) -> Result> -where - P: Clone, -{ - match obj { - Err(_py_err) => Err(EvalErr(err_node.clone(), msg.to_string())), - Ok(o) => Ok(o), - } -} diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index 07b29052..9e46f9c7 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -55,28 +55,14 @@ impl PyArena { py: Python<'p>, obj: &'p PyAny, ) -> PyResult<&'p PyCell> { - let borrowed_arena = slf.borrow(); - let ptr = borrowed_arena.ptr_for_obj(py, obj)?; + let ptr = Self::ptr_for_obj(slf, py, obj)?; PyCell::new(py, ArenaObject::new(py, slf, ptr)) } - pub fn ptr_for_obj<'p>(&'p self, py: Python<'p>, obj: &'p PyAny) -> PyResult { - let allocator: &mut IntAllocator = &mut self.allocator() as &mut IntAllocator; - - /* - let arena_obj: PyResult> = obj.extract(); - if let Ok(arena) = arena_obj { - let same_arena: bool = { - let o1: PyObject = self; - o1 == arena.borrow().get_arena(py)?.into() - }; - if same_arena { - return Ok(arena); - } - }; - */ - - self.populate_native(py, obj, allocator) + pub fn ptr_for_obj<'p>(slf: &PyCell, py: Python<'p>, obj: &'p PyAny) -> PyResult { + let arena = slf.borrow(); + let allocator: &mut IntAllocator = &mut arena.allocator() as &mut IntAllocator; + Self::populate_native(slf, py, obj, allocator) } } @@ -117,7 +103,7 @@ impl PyArena { } fn populate_native( - &self, + slf: &PyCell, py: Python, obj: &PyAny, allocator: &mut IntAllocator, @@ -128,8 +114,10 @@ impl PyArena { let mut pending: HashSet = HashSet::new(); apply_to_tree(obj, move |obj| { + let celf = slf.borrow(); + // is it in cache yet? - if self.from_py_to_native_cache(py, obj).is_ok() { + if celf.from_py_to_native_cache(py, obj).is_ok() { // yep, we're done return Ok(None); } @@ -140,7 +128,7 @@ impl PyArena { PyView::Atom(atom) => { let blob: &[u8] = atom.extract(py).unwrap(); let ptr = allocator.new_atom(blob).unwrap(); - self.add(py, obj, &ptr)?; + celf.add(py, obj, &ptr)?; Ok(None) } @@ -149,14 +137,14 @@ impl PyArena { let pair: &PyTuple = pair.extract()?; let p0: &PyAny = pair.get_item(0); let p1: &PyAny = pair.get_item(1); - let ptr_0: PyResult = self.from_py_to_native_cache(py, p0); - let ptr_1: PyResult = self.from_py_to_native_cache(py, p1); + let ptr_0: PyResult = celf.from_py_to_native_cache(py, p0); + let ptr_1: PyResult = celf.from_py_to_native_cache(py, p1); let as_obj = id_for_pyany(py, obj)?; if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); - self.add(py, obj, &ptr)?; + celf.add(py, obj, &ptr)?; pending.remove(&as_obj); Ok(None) @@ -178,17 +166,18 @@ impl PyArena { } })?; - self.from_py_to_native_cache(py, obj) + slf.borrow().from_py_to_native_cache(py, obj) } pub fn native_for_py( - &self, + slf: &PyCell, py: Python, obj: &PyAny, allocator: &mut IntAllocator, ) -> PyResult<::Ptr> { - self.from_py_to_native_cache(py, obj) - .or_else(|_err| self.populate_native(py, obj, allocator)) + let celf = slf.borrow(); + celf.from_py_to_native_cache(py, obj) + .or_else(|_err| Self::populate_native(slf, py, obj, allocator)) } // native to py methods From c6c91b1d4a6fafc725a463eed5ed7a8442452a21 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 12 May 2021 15:31:28 -0700 Subject: [PATCH 60/76] Get rid of `CLVMObject` in native code. --- src/py/api.rs | 5 ++- src/py/arena_object.rs | 8 ++--- src/py/clvm_object.rs | 78 ------------------------------------------ src/py/dialect.rs | 3 +- src/py/mod.rs | 1 - src/py/py_arena.rs | 17 ++++----- src/py/py_view.rs | 18 ++-------- src/py/run_program.rs | 7 ++-- 8 files changed, 19 insertions(+), 118 deletions(-) delete mode 100644 src/py/clvm_object.rs diff --git a/src/py/api.rs b/src/py/api.rs index 570d688f..4cd656e6 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -148,12 +148,11 @@ pub fn py_run_program<'p>( match r { Ok(reduction) => { - let r = arena.borrow().py_for_native(py, &reduction.1, allocator)?; + let r = arena_borrowed.py_for_native(py, &reduction.1, allocator)?; Ok((reduction.0, r.to_object(py))) } Err(eval_err) => { - let node: PyObject = arena - .borrow() + let node: PyObject = arena_borrowed .py_for_native(py, &eval_err.0, allocator)? .to_object(py); let s: String = eval_err.1; diff --git a/src/py/arena_object.rs b/src/py/arena_object.rs index 36026738..e62b7910 100644 --- a/src/py/arena_object.rs +++ b/src/py/arena_object.rs @@ -4,7 +4,6 @@ use pyo3::prelude::{pyclass, pymethods}; use pyo3::PyAny; use pyo3::PyCell; use pyo3::PyObject; -use pyo3::PyRef; use pyo3::PyResult; use pyo3::Python; use pyo3::ToPyObject; @@ -29,9 +28,10 @@ impl ArenaObject { #[pymethods] impl ArenaObject { fn clvm_object<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { - let arena: PyRef = self.arena.extract(py)?; - let mut allocator: RefMut = arena.allocator(); - arena.py_for_native(py, &self.ptr, &mut allocator) + let arena: &PyCell = self.arena.extract(py)?; + let arena_ptr = arena.borrow(); + let mut allocator: RefMut = arena_ptr.allocator(); + arena_ptr.py_for_native(py, &self.ptr, &mut allocator) } #[getter(arena)] diff --git a/src/py/clvm_object.rs b/src/py/clvm_object.rs deleted file mode 100644 index 649e2a04..00000000 --- a/src/py/clvm_object.rs +++ /dev/null @@ -1,78 +0,0 @@ -use pyo3::prelude::*; -use pyo3::types::{PyBytes, PyTuple, PyType}; - -use super::py_view::PyView; - -#[pyclass(weakref, subclass)] -pub struct CLVMObject { - pub py_view: PyView, -} - -impl CLVMObject { - pub fn new(py: Python, py_view: PyView) -> PyResult<&PyCell> { - PyCell::new(py, CLVMObject { py_view }) - } -} - -#[pymethods] -impl CLVMObject { - #[new] - fn new_obj(py: Python, obj: &PyAny) -> PyResult { - Ok(if let Ok(tuple) = obj.extract() { - let py_view = PyView::new_pair(py, tuple)?; - Self { py_view } - } else { - let py_bytes: &PyBytes = obj.extract()?; - let py_view = PyView::new_atom(py, py_bytes); - Self { py_view } - }) - } - - #[classmethod] - fn new_atom<'p>(_cls: &PyType, py: Python<'p>, atom: &PyBytes) -> PyResult<&'p PyCell> { - let py_view = PyView::new_atom(py, atom); - Self::new(py, py_view) - } - - #[classmethod] - fn new_pair<'p>( - _cls: &PyType, - py: Python<'p>, - p1: &PyCell, - p2: &PyCell, - ) -> PyResult<&'p PyCell> { - let tuple = PyTuple::new(py, &[p1, p2]); - let py_view = PyView::new_pair(py, tuple)?; - Self::new(py, py_view) - } - - #[classmethod] - fn new_tuple<'p>(_cls: &PyType, py: Python<'p>, tuple: &PyTuple) -> PyResult<&'p PyCell> { - let py_view = PyView::new_pair(py, tuple)?; - Self::new(py, py_view) - } - - #[getter(atom)] - pub fn atom<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - match &slf.try_borrow()?.py_view { - PyView::Atom(obj) => Ok(obj.clone()), - _ => Ok(py.None()), - } - } - - #[getter(pair)] - pub fn pair<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - match &slf.try_borrow()?.py_view { - PyView::Pair(obj) => Ok(obj.clone()), - _ => Ok(py.None()), - } - } - - #[getter(python)] - pub fn python<'p>(slf: &'p PyCell, py: Python<'p>) -> PyResult { - Ok(match &slf.borrow().py_view { - PyView::Atom(atom) => ("Atom", atom).to_object(py), - PyView::Pair(pair) => ("Pair", pair).to_object(py), - }) - } -} diff --git a/src/py/dialect.rs b/src/py/dialect.rs index da34b6e5..7431f9ee 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -226,8 +226,7 @@ impl Dialect { Ok((reduction.0, r)) } Err(eval_err) => { - let node: PyObject = arena - .borrow() + let node: PyObject = borrowed_arena .py_for_native(py, &eval_err.0, allocator)? .to_object(py); let s: String = eval_err.1; diff --git a/src/py/mod.rs b/src/py/mod.rs index 7c3a5c40..777de88b 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -1,6 +1,5 @@ pub mod api; pub mod arena_object; -pub mod clvm_object; pub mod dialect; pub mod error_bridge; pub mod f_table; diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index 9e46f9c7..d30aa004 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -10,24 +10,19 @@ use crate::int_allocator::IntAllocator; use crate::serialize::node_from_bytes; use super::arena_object::ArenaObject; -use super::clvm_object::CLVMObject; use super::py_view::PyView; +type Bridge = dyn for<'p> Fn(Python<'p>, &'p PyAny) -> PyResult<&'p PyAny>; + #[pyclass(subclass, unsendable)] pub struct PyArena { arena: RefCell, cache: PyObject, - bridge_constructor: Box Fn(Python<'p>, &'p PyAny) -> PyResult<&'p PyAny>>, + bridge_constructor: Box, } -fn clvm_obj_bridge<'p>(py: Python<'p>, obj: &'p PyAny) -> PyResult<&'p PyAny> { - if let Ok(b) = obj.extract::<&PyBytes>() { - Ok(CLVMObject::new(py, PyView::new_atom(py, b))?) - } else if let Ok(p) = obj.extract::<&PyTuple>() { - Ok(CLVMObject::new(py, PyView::new_pair(py, p)?)?) - } else { - panic!("bad bridge") - } +fn default_bridge<'p>(_py: Python<'p>, obj: &'p PyAny) -> PyResult<&'p PyAny> { + Ok(obj) } #[pymethods] @@ -37,7 +32,7 @@ impl PyArena { Ok(PyArena { arena: RefCell::new(IntAllocator::default()), cache: py.eval("dict()", None, None)?.to_object(py), - bridge_constructor: Box::new(clvm_obj_bridge), + bridge_constructor: Box::new(default_bridge), }) } diff --git a/src/py/py_view.rs b/src/py/py_view.rs index 9b9ee280..a86ec958 100644 --- a/src/py/py_view.rs +++ b/src/py/py_view.rs @@ -1,5 +1,4 @@ -use pyo3::types::{PyBytes, PyTuple}; -use pyo3::{PyAny, PyObject, PyResult, Python, ToPyObject}; +use pyo3::{PyAny, PyObject, PyResult}; #[derive(Clone)] pub enum PyView { @@ -8,20 +7,7 @@ pub enum PyView { } impl PyView { - pub fn new_atom(py: Python, atom: &PyBytes) -> Self { - PyView::Atom(atom.to_object(py)) - } - - pub fn new_pair(py: Python, pair: &PyTuple) -> PyResult { - if pair.len() != 2 { - py.eval("raise ValueError('new_pair requires 2-tuple')", None, None)?; - } - let _p0: &PyAny = pair.get_item(0).extract()?; - let _p1: &PyAny = pair.get_item(1).extract()?; - - Ok(PyView::Pair(pair.to_object(py))) - } - + // TODO: move this into `py_arena` and use very similar `SExp<'p, &'p PyBytes, &'p PyAny>` instead pub fn py_view_for_obj(obj: &PyAny) -> PyResult { let r: &PyAny = obj.getattr("atom")?.extract()?; if !r.is_none() { diff --git a/src/py/run_program.rs b/src/py/run_program.rs index b284b652..b709dac3 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -58,8 +58,9 @@ pub fn deserialize_and_run_program( max_cost: Cost, flags: u32, ) -> PyResult<(Cost, PyObject)> { - let arena = PyArena::new(py)?; - let mut allocator_refcell: RefMut = arena.allocator(); + let arena = PyArena::new_cell(py)?; + let arena_borrowed = arena.borrow(); + let mut allocator_refcell: RefMut = arena_borrowed.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; let f_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); let strict: bool = flags != 0; @@ -75,7 +76,7 @@ pub fn deserialize_and_run_program( match r { Ok(reduction) => Ok(( reduction.0, - arena + arena_borrowed .py_for_native(py, &reduction.1, allocator)? .to_object(py), )), From 2379a6acb092fb6df3cecc6f1eebf0c517849335 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 12 May 2021 17:01:27 -0700 Subject: [PATCH 61/76] Many `clvm_tools` tests now pass. --- src/py/dialect.rs | 8 +++++--- src/py/py_arena.rs | 22 ++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 7431f9ee..14b3fcd8 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -136,8 +136,9 @@ impl Dialect { args: &PyAny, max_cost: Cost, pre_eval_f: &PyAny, + to_python: &PyAny, ) -> PyResult<(Cost, PyObject)> { - let arena = PyArena::new_cell(py)?; + let arena = PyArena::new_cell_obj(py, to_python.to_object(py))?; let arena_ptr: &PyArena = &arena.borrow() as &PyArena; let program = PyArena::ptr_for_obj(arena, py, program)?; @@ -175,8 +176,9 @@ impl Dialect { args_blob: &[u8], max_cost: Cost, pre_eval: &PyAny, + to_python: &PyAny, ) -> PyResult<(Cost, ArenaObject)> { - let arena = PyArena::new_cell(py)?; + let arena = PyArena::new_cell_obj(py, to_python.to_object(py))?; let (program, args) = { let arena_ptr: &PyArena = &arena.borrow() as &PyArena; let mut allocator_refcell: RefMut = arena_ptr.allocator(); @@ -268,7 +270,7 @@ impl DialectRunningContext<'_> { let eval_err: PyResult> = eval_err_for_pyerr(py, &pyerr, self.arena, allocator); let r: EvalErr = - unwrap_or_eval_err(eval_err, args, "2unexpected exception")?; + unwrap_or_eval_err(eval_err, args, "unexpected exception")?; Err(r) } Ok(o) => { diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index d30aa004..0a3cdb05 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -12,27 +12,21 @@ use crate::serialize::node_from_bytes; use super::arena_object::ArenaObject; use super::py_view::PyView; -type Bridge = dyn for<'p> Fn(Python<'p>, &'p PyAny) -> PyResult<&'p PyAny>; - #[pyclass(subclass, unsendable)] pub struct PyArena { arena: RefCell, cache: PyObject, - bridge_constructor: Box, -} - -fn default_bridge<'p>(_py: Python<'p>, obj: &'p PyAny) -> PyResult<&'p PyAny> { - Ok(obj) + to_python: PyObject, } #[pymethods] impl PyArena { #[new] - pub fn new(py: Python) -> PyResult { + pub fn new(py: Python, new_obj_f: PyObject) -> PyResult { Ok(PyArena { arena: RefCell::new(IntAllocator::default()), cache: py.eval("dict()", None, None)?.to_object(py), - bridge_constructor: Box::new(default_bridge), + to_python: new_obj_f, }) } @@ -62,8 +56,12 @@ impl PyArena { } impl PyArena { + pub fn new_cell_obj(py: Python, new_obj_f: PyObject) -> PyResult<&PyCell> { + PyCell::new(py, PyArena::new(py, new_obj_f)?) + } + pub fn new_cell(py: Python) -> PyResult<&PyCell> { - PyCell::new(py, PyArena::new(py)?) + Self::new_cell_obj(py, py.eval("lambda sexp: sexp", None, None)?.to_object(py)) } pub fn allocator(&self) -> RefMut { @@ -206,7 +204,7 @@ impl PyArena { // it's an atom, so we just populate cache directly let blob = allocator.buf(&a); let py_bytes = PyBytes::new(py, blob); - self.add(py, (self.bridge_constructor)(py, py_bytes)?, &ptr)?; + self.add(py, self.to_python.as_ref(py).call1((py_bytes,))?, &ptr)?; Ok(None) } SExp::Pair(ptr_1, ptr_2) => { @@ -229,7 +227,7 @@ impl PyArena { // the children are in the cache, create new node & populate cache with it Ok(tuple) => { let (_p1, _p2): (&PyAny, &PyAny) = tuple.extract()?; - self.add(py, (self.bridge_constructor)(py, tuple)?, &ptr)?; + self.add(py, self.to_python.as_ref(py).call1((tuple,))?, &ptr)?; Ok(None) } } From 16f08321d8c06a4fb9532d8bd272287e921cff45 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 12 May 2021 18:41:38 -0700 Subject: [PATCH 62/76] Ditch `ArenaObject`. --- src/py/api.rs | 2 -- src/py/arena_object.rs | 62 ------------------------------------------ src/py/dialect.rs | 48 ++++++-------------------------- src/py/mod.rs | 1 - src/py/native_op.rs | 20 ++++++++------ src/py/py_arena.rs | 53 ++++++++++++++++-------------------- 6 files changed, 44 insertions(+), 142 deletions(-) delete mode 100644 src/py/arena_object.rs diff --git a/src/py/api.rs b/src/py/api.rs index 4cd656e6..16bacbb9 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -15,7 +15,6 @@ use crate::node::Node; use crate::reduction::{EvalErr, Reduction}; use crate::serialize::node_to_bytes; -use super::arena_object::ArenaObject; use super::dialect::{Dialect, PyMultiOpFn}; use super::f_table::OpFn; use super::native_op::NativeOp; @@ -99,7 +98,6 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyAny) -> PyResult<&'p PyBytes> /// This module is a python module implemented in Rust. #[pymodule] fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/py/arena_object.rs b/src/py/arena_object.rs deleted file mode 100644 index e62b7910..00000000 --- a/src/py/arena_object.rs +++ /dev/null @@ -1,62 +0,0 @@ -use std::cell::RefMut; - -use pyo3::prelude::{pyclass, pymethods}; -use pyo3::PyAny; -use pyo3::PyCell; -use pyo3::PyObject; -use pyo3::PyResult; -use pyo3::Python; -use pyo3::ToPyObject; - -use crate::int_allocator::IntAllocator; - -use super::py_arena::PyArena; - -#[pyclass(weakref, subclass)] -pub struct ArenaObject { - arena: PyObject, - ptr: i32, -} - -impl ArenaObject { - pub fn new(py: Python, arena: &PyCell, ptr: i32) -> Self { - let arena = arena.to_object(py); - Self { arena, ptr } - } -} - -#[pymethods] -impl ArenaObject { - fn clvm_object<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { - let arena: &PyCell = self.arena.extract(py)?; - let arena_ptr = arena.borrow(); - let mut allocator: RefMut = arena_ptr.allocator(); - arena_ptr.py_for_native(py, &self.ptr, &mut allocator) - } - - #[getter(arena)] - pub fn get_arena<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyCell> { - self.arena.extract(py) - } - - #[getter(ptr)] - pub fn get_ptr(&self) -> i32 { - self.ptr - } - - #[getter(atom)] - pub fn get_atom<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { - self.clvm_object(py)?.getattr("atom") - } - - #[getter(pair)] - pub fn get_pair<'p>(&'p self, py: Python<'p>) -> PyResult<&'p PyAny> { - self.clvm_object(py)?.getattr("pair") - } -} - -impl From<&ArenaObject> for i32 { - fn from(obj: &ArenaObject) -> Self { - obj.ptr - } -} diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 14b3fcd8..ebb6aee9 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -3,7 +3,7 @@ use std::collections::HashMap; use pyo3::prelude::{pyclass, pymethods}; -use pyo3::types::{PyBytes, PyString, PyTuple}; +use pyo3::types::{PyString, PyTuple}; use pyo3::{FromPyObject, PyAny, PyCell, PyObject, PyRef, PyResult, Python, ToPyObject}; use crate::allocator::Allocator; @@ -15,7 +15,6 @@ use crate::reduction::Response; use crate::run_program::OperatorHandler; use crate::serialize::node_from_bytes; -use super::arena_object::ArenaObject; use super::error_bridge::{eval_err_for_pyerr, raise_eval_error, unwrap_or_eval_err}; use super::f_table::FLookup; use super::f_table::OpFn; @@ -75,12 +74,6 @@ impl<'source> FromPyObject<'source> for MultiOpFnE { } } -fn same_arena(arena1: &PyArena, arena2: &PyArena) -> bool { - let p1: *const PyArena = arena1 as *const PyArena; - let p2: *const PyArena = arena2 as *const PyArena; - p1 == p2 -} - #[pyclass] pub struct Dialect { quote_kw: Vec, @@ -141,32 +134,11 @@ impl Dialect { let arena = PyArena::new_cell_obj(py, to_python.to_object(py))?; let arena_ptr: &PyArena = &arena.borrow() as &PyArena; - let program = PyArena::ptr_for_obj(arena, py, program)?; - let args = PyArena::ptr_for_obj(arena, py, args)?; + let program = arena_ptr.ptr_for_obj(py, program)?; + let args = arena_ptr.ptr_for_obj(py, args)?; let (cost, r) = self.run_program_ptr(py, &arena, program, args, max_cost, pre_eval_f)?; - - let mut allocator_refcell: RefMut = arena_ptr.allocator(); - let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - - let r_ptr = &(&r).into(); - let new_r = arena_ptr.py_for_native(py, r_ptr, allocator)?; - Ok((cost, new_r.into())) - } - - pub fn run_program_arena<'p>( - &self, - py: Python<'p>, - program: &ArenaObject, - args: &ArenaObject, - max_cost: Cost, - pre_eval: &PyAny, - ) -> PyResult<(Cost, ArenaObject)> { - let arena = program.get_arena(py)?; - if !same_arena(&arena.borrow(), &args.get_arena(py)?.borrow()) { - py.eval("raise ValueError('mismatched arenas')", None, None)?; - } - self.run_program_ptr(py, arena, program.into(), args.into(), max_cost, pre_eval) + Ok((cost, r.to_object(py))) } pub fn deserialize_and_run_program<'p>( @@ -177,7 +149,7 @@ impl Dialect { max_cost: Cost, pre_eval: &PyAny, to_python: &PyAny, - ) -> PyResult<(Cost, ArenaObject)> { + ) -> PyResult<(Cost, &'p PyAny)> { let arena = PyArena::new_cell_obj(py, to_python.to_object(py))?; let (program, args) = { let arena_ptr: &PyArena = &arena.borrow() as &PyArena; @@ -196,12 +168,12 @@ impl Dialect { pub fn run_program_ptr<'p>( &self, py: Python<'p>, - arena: &PyCell, + arena: &'p PyCell, program: i32, args: i32, max_cost: Cost, _pre_eval: &PyAny, - ) -> PyResult<(Cost, ArenaObject)> { + ) -> PyResult<(Cost, &'p PyAny)> { let borrowed_arena = arena.borrow(); let mut allocator_refcell: RefMut = borrowed_arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; @@ -224,7 +196,7 @@ impl Dialect { match r { Ok(reduction) => { - let r = ArenaObject::new(py, arena, reduction.1); + let r = borrowed_arena.py_for_native(py, &reduction.1, allocator)?; Ok((reduction.0, r)) } Err(eval_err) => { @@ -253,12 +225,10 @@ impl DialectRunningContext<'_> { &self, obj: &PyObject, allocator: &mut IntAllocator, - op_buf: ::AtomBuf, args: &::Ptr, max_cost: Cost, ) -> Response<::Ptr> { Python::with_gil(|py| { - let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); let r = unwrap_or_eval_err( PyArena::py_for_native(&self.arena.borrow(), py, args, allocator), args, @@ -309,7 +279,7 @@ impl OperatorHandler for DialectRunningContext<'_> { if let Some(op_fn) = self.dialect.native_u8_lookup.get(op) { op_fn(allocator, *argument_list, max_cost) } else if let Some(op_fn) = self.dialect.python_u8_lookup.get(op) { - self.invoke_py_obj(op_fn, allocator, o, argument_list, max_cost) + self.invoke_py_obj(op_fn, allocator, argument_list, max_cost) } else { self.dialect .unknown_op_callback diff --git a/src/py/mod.rs b/src/py/mod.rs index 777de88b..1d127641 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -1,5 +1,4 @@ pub mod api; -pub mod arena_object; pub mod dialect; pub mod error_bridge; pub mod f_table; diff --git a/src/py/native_op.rs b/src/py/native_op.rs index 333ea2f5..c7952f1d 100644 --- a/src/py/native_op.rs +++ b/src/py/native_op.rs @@ -1,12 +1,11 @@ use pyo3::prelude::{pyclass, pymethods}; use pyo3::types::PyString; -use pyo3::{PyAny, PyCell, PyResult, Python, ToPyObject}; +use pyo3::{PyAny, PyObject, PyResult, Python, ToPyObject}; use crate::cost::Cost; use crate::int_allocator::IntAllocator; use crate::reduction::Reduction; -use super::arena_object::ArenaObject; use super::error_bridge::raise_eval_error; use super::f_table::OpFn; use super::py_arena::PyArena; @@ -27,24 +26,29 @@ impl NativeOp { #[pymethods] impl NativeOp { #[call] - fn __call__(&self, py: Python, args: &PyAny, _max_cost: Cost) -> PyResult<(Cost, ArenaObject)> { + fn __call__<'p>( + &'p self, + py: Python<'p>, + args: &'p PyAny, + _max_cost: Cost, + ) -> PyResult<(Cost, PyObject)> { let arena_cell = PyArena::new_cell(py)?; - let ptr = PyArena::include(arena_cell, py, args)?.borrow().get_ptr(); let arena: &PyArena = &arena_cell.borrow(); + let ptr = arena.ptr_for_obj(py, args)?; let mut allocator = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator; let r = (self.op)(allocator, ptr, _max_cost); match r { Ok(Reduction(cost, ptr)) => { - let r = ArenaObject::new(py, arena_cell, ptr); - Ok((cost, r)) + let r = arena.obj_for_ptr(py, ptr)?; + Ok((cost, r.to_object(py))) } Err(_err) => { - let r = ArenaObject::new(py, arena_cell, ptr); + let r = arena.obj_for_ptr(py, ptr)?; match raise_eval_error( py, PyString::new(py, "problem in suboperator"), - PyCell::new(py, r)?.to_object(py), + r.to_object(py), ) { Err(e) => Err(e), Ok(_) => panic!("oh dear"), diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index 0a3cdb05..b936d1d8 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -9,7 +9,6 @@ use crate::allocator::{Allocator, SExp}; use crate::int_allocator::IntAllocator; use crate::serialize::node_from_bytes; -use super::arena_object::ArenaObject; use super::py_view::PyView; #[pyclass(subclass, unsendable)] @@ -30,30 +29,22 @@ impl PyArena { }) } - pub fn deserialize(slf: &PyCell, py: Python, blob: &[u8]) -> PyResult { - let ptr = { - let borrowed_arena = slf.borrow(); - let allocator: &mut IntAllocator = &mut borrowed_arena.allocator() as &mut IntAllocator; - node_from_bytes(allocator, blob)? - }; - Ok(ArenaObject::new(py, slf, ptr)) + pub fn deserialize<'p>(&self, py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyAny> { + let allocator: &mut IntAllocator = &mut self.allocator() as &mut IntAllocator; + let ptr = node_from_bytes(allocator, blob)?; + self.py_for_native(py, &ptr, allocator) } - pub fn include<'p>( - slf: &'p PyCell, - py: Python<'p>, - obj: &'p PyAny, - ) -> PyResult<&'p PyCell> { - let ptr = Self::ptr_for_obj(slf, py, obj)?; - PyCell::new(py, ArenaObject::new(py, slf, ptr)) + pub fn include<'p>(&self, py: Python<'p>, obj: &'p PyAny) -> PyResult<&'p PyAny> { + let ptr = Self::ptr_for_obj(self, py, obj)?; + self.py_for_native(py, &ptr, &mut self.allocator() as &mut IntAllocator) } - pub fn ptr_for_obj<'p>(slf: &PyCell, py: Python<'p>, obj: &'p PyAny) -> PyResult { - let arena = slf.borrow(); - let allocator: &mut IntAllocator = &mut arena.allocator() as &mut IntAllocator; - Self::populate_native(slf, py, obj, allocator) + pub fn ptr_for_obj(&self, py: Python, obj: &PyAny) -> PyResult { + let allocator: &mut IntAllocator = &mut self.allocator() as &mut IntAllocator; + self.populate_native(py, obj, allocator) } -} + } impl PyArena { pub fn new_cell_obj(py: Python, new_obj_f: PyObject) -> PyResult<&PyCell> { @@ -64,6 +55,10 @@ impl PyArena { Self::new_cell_obj(py, py.eval("lambda sexp: sexp", None, None)?.to_object(py)) } + pub fn obj_for_ptr<'p>(&self, py: Python<'p>, ptr: i32) -> PyResult<&'p PyAny> { + self.py_for_native(py, &ptr, &mut self.allocator()) + } + pub fn allocator(&self) -> RefMut { self.arena.borrow_mut() } @@ -96,7 +91,7 @@ impl PyArena { } fn populate_native( - slf: &PyCell, + &self, py: Python, obj: &PyAny, allocator: &mut IntAllocator, @@ -107,10 +102,8 @@ impl PyArena { let mut pending: HashSet = HashSet::new(); apply_to_tree(obj, move |obj| { - let celf = slf.borrow(); - // is it in cache yet? - if celf.from_py_to_native_cache(py, obj).is_ok() { + if self.from_py_to_native_cache(py, obj).is_ok() { // yep, we're done return Ok(None); } @@ -121,7 +114,7 @@ impl PyArena { PyView::Atom(atom) => { let blob: &[u8] = atom.extract(py).unwrap(); let ptr = allocator.new_atom(blob).unwrap(); - celf.add(py, obj, &ptr)?; + self.add(py, obj, &ptr)?; Ok(None) } @@ -130,14 +123,14 @@ impl PyArena { let pair: &PyTuple = pair.extract()?; let p0: &PyAny = pair.get_item(0); let p1: &PyAny = pair.get_item(1); - let ptr_0: PyResult = celf.from_py_to_native_cache(py, p0); - let ptr_1: PyResult = celf.from_py_to_native_cache(py, p1); + let ptr_0: PyResult = self.from_py_to_native_cache(py, p0); + let ptr_1: PyResult = self.from_py_to_native_cache(py, p1); let as_obj = id_for_pyany(py, obj)?; if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); - celf.add(py, obj, &ptr)?; + self.add(py, obj, &ptr)?; pending.remove(&as_obj); Ok(None) @@ -159,7 +152,7 @@ impl PyArena { } })?; - slf.borrow().from_py_to_native_cache(py, obj) + self.from_py_to_native_cache(py, obj) } pub fn native_for_py( @@ -170,7 +163,7 @@ impl PyArena { ) -> PyResult<::Ptr> { let celf = slf.borrow(); celf.from_py_to_native_cache(py, obj) - .or_else(|_err| Self::populate_native(slf, py, obj, allocator)) + .or_else(|_err| celf.populate_native(py, obj, allocator)) } // native to py methods From 5f1c7134a3fc3ed8f49602cd87a452a22d09ded4 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 12 May 2021 18:47:08 -0700 Subject: [PATCH 63/76] Delete a bunch of stuff. --- src/py/api.rs | 68 +----------------------------- src/py/mod.rs | 1 - src/py/op_fn.rs | 100 --------------------------------------------- src/py/py_arena.rs | 2 +- 4 files changed, 2 insertions(+), 169 deletions(-) delete mode 100644 src/py/op_fn.rs diff --git a/src/py/api.rs b/src/py/api.rs index 16bacbb9..af89e12c 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -1,24 +1,20 @@ use std::cell::RefMut; -use std::collections::HashMap; use pyo3::prelude::*; -use pyo3::types::{PyBytes, PyDict, PyString}; +use pyo3::types::{PyBytes, PyDict}; use pyo3::wrap_pyfunction; use pyo3::PyObject; use crate::core_ops::*; -use crate::cost::Cost; use crate::err_utils::err; use crate::int_allocator::IntAllocator; use crate::more_ops::*; use crate::node::Node; -use crate::reduction::{EvalErr, Reduction}; use crate::serialize::node_to_bytes; use super::dialect::{Dialect, PyMultiOpFn}; use super::f_table::OpFn; use super::native_op::NativeOp; -use super::op_fn::PyOperatorHandler; use super::py_arena::PyArena; use super::run_program::{ __pyo3_get_function_deserialize_and_run_program, __pyo3_get_function_serialized_length, @@ -65,22 +61,6 @@ pub fn native_opcodes_dict(py: Python) -> PyResult { Ok(r.to_object(py)) } -#[pyfunction] -fn raise_eval_error(py: Python, msg: &PyString, sexp: PyObject) -> PyResult { - let ctx: &PyDict = PyDict::new(py); - ctx.set_item("msg", msg)?; - ctx.set_item("sexp", sexp)?; - let r = py.run( - "from clvm.EvalError import EvalError; raise EvalError(msg, sexp)", - None, - Some(ctx), - ); - match r { - Err(x) => Err(x), - Ok(_) => Ok(ctx.into()), - } -} - #[pyfunction] fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyAny) -> PyResult<&'p PyBytes> { let arena = PyArena::new_cell(py)?; @@ -112,54 +92,8 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add("NATIVE_OP_UNKNOWN_NON_STRICT", PyMultiOpFn::new(op_unknown))?; - m.add_function(wrap_pyfunction!(py_run_program, m)?)?; m.add_function(wrap_pyfunction!(serialize_to_bytes, m)?)?; m.add_function(wrap_pyfunction!(deserialize_and_run_program, m)?)?; Ok(()) } - -#[pyfunction] -#[allow(clippy::too_many_arguments)] -pub fn py_run_program<'p>( - py: Python<'p>, - program: &PyAny, - args: &PyAny, - quote_kw: &[u8], - apply_kw: &[u8], - max_cost: Cost, - opcode_lookup_by_name: HashMap>, - py_callback: PyObject, -) -> PyResult<(Cost, PyObject)> { - let arena = PyArena::new_cell(py)?; - let arena_borrowed = arena.borrow(); - let mut allocator_refcell: RefMut = arena_borrowed.allocator(); - let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - - let op_lookup = PyOperatorHandler::new(opcode_lookup_by_name, py_callback, arena)?; - let program = PyArena::native_for_py(arena, py, program, allocator)?; - let args = PyArena::native_for_py(arena, py, args, allocator)?; - - let r: Result, EvalErr> = crate::run_program::run_program( - allocator, &program, &args, quote_kw, apply_kw, max_cost, &op_lookup, None, - ); - - match r { - Ok(reduction) => { - let r = arena_borrowed.py_for_native(py, &reduction.1, allocator)?; - Ok((reduction.0, r.to_object(py))) - } - Err(eval_err) => { - let node: PyObject = arena_borrowed - .py_for_native(py, &eval_err.0, allocator)? - .to_object(py); - let s: String = eval_err.1; - let s1: &str = &s; - let msg: &PyString = PyString::new(py, s1); - match raise_eval_error(py, &msg, node) { - Err(x) => Err(x), - _ => panic!(), - } - } - } -} diff --git a/src/py/mod.rs b/src/py/mod.rs index 1d127641..8e5b1e89 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -3,7 +3,6 @@ pub mod dialect; pub mod error_bridge; pub mod f_table; pub mod native_op; -pub mod op_fn; pub mod py_arena; pub mod py_view; pub mod run_program; diff --git a/src/py/op_fn.rs b/src/py/op_fn.rs deleted file mode 100644 index f4cdd864..00000000 --- a/src/py/op_fn.rs +++ /dev/null @@ -1,100 +0,0 @@ -use std::collections::HashMap; - -use pyo3::types::PyBytes; -use pyo3::types::PyTuple; -use pyo3::PyObject; -use pyo3::PyResult; -use pyo3::Python; -use pyo3::ToPyObject; -use pyo3::{PyAny, PyCell}; - -use crate::allocator::Allocator; -use crate::cost::Cost; -use crate::int_allocator::IntAllocator; -use crate::reduction::{EvalErr, Reduction, Response}; -use crate::run_program::OperatorHandler; - -use super::error_bridge::{eval_err_for_pyerr, unwrap_or_eval_err}; -use super::f_table::{f_lookup_for_hashmap, FLookup}; -use super::py_arena::PyArena; - -pub struct PyOperatorHandler<'p> { - native_lookup: FLookup, - py_callable: PyObject, - arena: &'p PyCell, -} - -impl<'p> PyOperatorHandler<'p> { - pub fn new( - opcode_lookup_by_name: HashMap>, - py_callable: PyObject, - arena: &'p PyCell, - ) -> PyResult { - let native_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); - Ok(PyOperatorHandler { - native_lookup, - py_callable, - arena, - }) - } - - pub fn invoke_py_obj( - &self, - obj: PyObject, - allocator: &mut IntAllocator, - op_buf: ::AtomBuf, - args: &::Ptr, - max_cost: Cost, - ) -> Response<::Ptr> { - Python::with_gil(|py| { - let op: &PyBytes = PyBytes::new(py, allocator.buf(&op_buf)); - let r = unwrap_or_eval_err( - PyArena::py_for_native(&self.arena.borrow(), py, args, allocator), - args, - "can't uncache", - )?; - let r1 = obj.call1(py, (op, r.to_object(py), max_cost)); - match r1 { - Err(pyerr) => { - let eval_err: PyResult> = - eval_err_for_pyerr(py, &pyerr, self.arena, allocator); - let r: EvalErr = - unwrap_or_eval_err(eval_err, args, "unexpected exception")?; - Err(r) - } - Ok(o) => { - let pair: &PyTuple = unwrap_or_eval_err(o.extract(py), args, "expected tuple")?; - - let i0: u32 = - unwrap_or_eval_err(pair.get_item(0).extract(), args, "expected u32")?; - - let clvm_object: &PyAny = - unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - - let r = PyArena::native_for_py(self.arena, py, clvm_object, allocator); - let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; - Ok(Reduction(i0 as Cost, node)) - } - } - }) - } -} - -impl OperatorHandler for PyOperatorHandler<'_> { - fn op( - &self, - allocator: &mut IntAllocator, - op_buf: ::AtomBuf, - args: &::Ptr, - max_cost: Cost, - ) -> Response<::Ptr> { - let op = allocator.buf(&op_buf); - if op.len() == 1 { - if let Some(f) = self.native_lookup[op[0] as usize] { - return f(allocator, *args, max_cost); - } - } - - self.invoke_py_obj(self.py_callable.clone(), allocator, op_buf, args, max_cost) - } -} diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index b936d1d8..dc41b403 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -44,7 +44,7 @@ impl PyArena { let allocator: &mut IntAllocator = &mut self.allocator() as &mut IntAllocator; self.populate_native(py, obj, allocator) } - } +} impl PyArena { pub fn new_cell_obj(py: Python, new_obj_f: PyObject) -> PyResult<&PyCell> { From e28a46291eced61871fda0c1d7deb17d6fed153b Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 13 May 2021 11:55:31 -0700 Subject: [PATCH 64/76] Fix a test. --- src/py/native_op.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/native_op.rs b/src/py/native_op.rs index c7952f1d..6caeddf0 100644 --- a/src/py/native_op.rs +++ b/src/py/native_op.rs @@ -40,11 +40,11 @@ impl NativeOp { let r = (self.op)(allocator, ptr, _max_cost); match r { Ok(Reduction(cost, ptr)) => { - let r = arena.obj_for_ptr(py, ptr)?; + let r = arena.py_for_native(py, &ptr, allocator)?; Ok((cost, r.to_object(py))) } Err(_err) => { - let r = arena.obj_for_ptr(py, ptr)?; + let r = arena.py_for_native(py, &ptr, allocator)?; match raise_eval_error( py, PyString::new(py, "problem in suboperator"), From dd376c4ffa0b38392febb3626bf122f8a8b13fe0 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 13 May 2021 12:41:12 -0700 Subject: [PATCH 65/76] Fix more tests. --- src/py/api.rs | 4 ++-- src/py/dialect.rs | 7 +++++-- src/run_program.rs | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index af89e12c..e78fde09 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -6,7 +6,7 @@ use pyo3::wrap_pyfunction; use pyo3::PyObject; use crate::core_ops::*; -use crate::err_utils::err; +use crate::err_utils::u8_err; use crate::int_allocator::IntAllocator; use crate::more_ops::*; use crate::node::Node; @@ -87,7 +87,7 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add( "NATIVE_OP_UNKNOWN_STRICT", - PyMultiOpFn::new(|_a, _b, op, _d| err(op, "unimplemented operator")), + PyMultiOpFn::new(|_a, b, _op, _d| u8_err(_a, &b, "unimplemented operator")), )?; m.add("NATIVE_OP_UNKNOWN_NON_STRICT", PyMultiOpFn::new(op_unknown))?; diff --git a/src/py/dialect.rs b/src/py/dialect.rs index ebb6aee9..7efafc98 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -12,7 +12,7 @@ use crate::int_allocator::IntAllocator; use crate::reduction::EvalErr; use crate::reduction::Reduction; use crate::reduction::Response; -use crate::run_program::OperatorHandler; +use crate::run_program::{PreEval, OperatorHandler}; use crate::serialize::node_from_bytes; use super::error_bridge::{eval_err_for_pyerr, raise_eval_error, unwrap_or_eval_err}; @@ -183,6 +183,9 @@ impl Dialect { arena: &arena, }; + let local_pre_eval: PreEval = + Box::new(|_allocator, _program, _args| Ok(None)); + let r: Result, EvalErr> = crate::run_program::run_program( allocator, &program, @@ -191,7 +194,7 @@ impl Dialect { &self.apply_kw, max_cost, &drc, - None, + Some(local_pre_eval), ); match r { diff --git a/src/run_program.rs b/src/run_program.rs index c7a5544c..05c57378 100644 --- a/src/run_program.rs +++ b/src/run_program.rs @@ -28,7 +28,7 @@ pub trait OperatorHandler { } pub type PreEval = Box< - dyn Fn( + fn( &mut A, &::Ptr, &::Ptr, From 334195400b4bef39df737f43149b103982756b20 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 14 May 2021 15:40:19 -0700 Subject: [PATCH 66/76] Require `to_python` in `Dialect` constructor. --- src/py/dialect.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 7efafc98..3504d80c 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -12,7 +12,7 @@ use crate::int_allocator::IntAllocator; use crate::reduction::EvalErr; use crate::reduction::Reduction; use crate::reduction::Response; -use crate::run_program::{PreEval, OperatorHandler}; +use crate::run_program::{OperatorHandler, PreEval}; use crate::serialize::node_from_bytes; use super::error_bridge::{eval_err_for_pyerr, raise_eval_error, unwrap_or_eval_err}; @@ -82,6 +82,7 @@ pub struct Dialect { python_u8_lookup: HashMap, PyObject>, native_u8_lookup: HashMap, OpFn>, unknown_op_callback: MultiOpFnE, + to_python: PyObject, } #[pymethods] @@ -91,6 +92,7 @@ impl Dialect { quote_kw: Vec, apply_kw: Vec, unknown_op_callback: MultiOpFnE, + to_python: PyObject, ) -> PyResult { let u8_lookup = [None; 256]; let python_u8_lookup = HashMap::new(); @@ -102,6 +104,7 @@ impl Dialect { python_u8_lookup, native_u8_lookup, unknown_op_callback, + to_python, }) } @@ -129,9 +132,8 @@ impl Dialect { args: &PyAny, max_cost: Cost, pre_eval_f: &PyAny, - to_python: &PyAny, ) -> PyResult<(Cost, PyObject)> { - let arena = PyArena::new_cell_obj(py, to_python.to_object(py))?; + let arena = PyArena::new_cell_obj(py, self.to_python.clone())?; let arena_ptr: &PyArena = &arena.borrow() as &PyArena; let program = arena_ptr.ptr_for_obj(py, program)?; @@ -148,9 +150,8 @@ impl Dialect { args_blob: &[u8], max_cost: Cost, pre_eval: &PyAny, - to_python: &PyAny, ) -> PyResult<(Cost, &'p PyAny)> { - let arena = PyArena::new_cell_obj(py, to_python.to_object(py))?; + let arena = PyArena::new_cell_obj(py, self.to_python.clone())?; let (program, args) = { let arena_ptr: &PyArena = &arena.borrow() as &PyArena; let mut allocator_refcell: RefMut = arena_ptr.allocator(); From 698c6bd1af5c02ca28e4fa5bf40039aeaf5b8b6f Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 18 May 2021 13:44:54 -0700 Subject: [PATCH 67/76] checkpoint --- src/py/dialect.rs | 41 ++++++++++++++++++++++++++++++++++++----- src/run_program.rs | 37 ++++++++++++++++++++----------------- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 3504d80c..e43871d4 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -149,7 +149,7 @@ impl Dialect { program_blob: &[u8], args_blob: &[u8], max_cost: Cost, - pre_eval: &PyAny, + pre_eval: &'p PyAny, ) -> PyResult<(Cost, &'p PyAny)> { let arena = PyArena::new_cell_obj(py, self.to_python.clone())?; let (program, args) = { @@ -173,7 +173,7 @@ impl Dialect { program: i32, args: i32, max_cost: Cost, - _pre_eval: &PyAny, + pre_eval: &'p PyAny, ) -> PyResult<(Cost, &'p PyAny)> { let borrowed_arena = arena.borrow(); let mut allocator_refcell: RefMut = borrowed_arena.allocator(); @@ -184,8 +184,39 @@ impl Dialect { arena: &arena, }; - let local_pre_eval: PreEval = - Box::new(|_allocator, _program, _args| Ok(None)); + let pre_eval_obj = pre_eval.to_object(py); + + let local_pre_eval: PreEval = Box::new(|allocator, program, args| { + { + // TODO: fix unwraps + let program_obj = borrowed_arena + .py_for_native(py, program, allocator) + .unwrap(); + let args_obj = borrowed_arena.py_for_native(py, args, allocator).unwrap(); + if !pre_eval_obj.is_none(py) { + let post_eval_obj = pre_eval_obj + .call1(py, (program_obj, args_obj)) + .unwrap() + .to_object(py); + } + + let local_arena = arena.to_object(py); + /* + let post_eval: Box> = + Box::new(move |allocator: &mut IntAllocator, result_ptr: &i32| { + dbg!("** 1"); + let arena: PyRef = local_arena.extract(py).unwrap(); + dbg!("** 1"); + let r = arena.py_for_native(py, &result_ptr, allocator).unwrap(); + dbg!("** 1"); + post_eval_obj.call1(py, (r,)).unwrap(); + dbg!("** 1"); + }); + */ + Ok(None) + } + }); + let pre_eval_f = Some(local_pre_eval); let r: Result, EvalErr> = crate::run_program::run_program( allocator, @@ -195,7 +226,7 @@ impl Dialect { &self.apply_kw, max_cost, &drc, - Some(local_pre_eval), + pre_eval_f, ); match r { diff --git a/src/run_program.rs b/src/run_program.rs index 05c57378..ef4967a1 100644 --- a/src/run_program.rs +++ b/src/run_program.rs @@ -27,15 +27,16 @@ pub trait OperatorHandler { ) -> Response<::Ptr>; } -pub type PreEval = Box< - fn( - &mut A, - &::Ptr, - &::Ptr, - ) -> Result>>, EvalErr<::Ptr>>, +pub type PreEval<'a, A> = Box< + dyn Fn( + &mut A, + &::Ptr, + &::Ptr, + ) -> Result>>, EvalErr<::Ptr>> + + 'a, >; -pub type PostEval = dyn Fn(Option<&::Ptr>); +pub type PostEval<'a, T> = dyn Fn(&mut T, &::Ptr) -> () + 'a; #[repr(u8)] enum Operation { @@ -54,8 +55,8 @@ pub struct RunProgramContext<'a, T: Allocator> { quote_kw: &'a [u8], apply_kw: &'a [u8], operator_lookup: &'a dyn OperatorHandler, - pre_eval: Option>, - posteval_stack: Vec>>, + pre_eval: Option>, + posteval_stack: Vec>>, val_stack: Vec, op_stack: Vec, } @@ -162,7 +163,7 @@ impl<'a, 'h, T: Allocator> RunProgramContext<'a, T> { quote_kw: &'a [u8], apply_kw: &'a [u8], operator_lookup: &'a dyn OperatorHandler, - pre_eval: Option>, + pre_eval: Option>, ) -> Self { RunProgramContext { allocator, @@ -357,7 +358,9 @@ impl<'a, T: Allocator> RunProgramContext<'a, T> { Operation::PostEval => { let f = self.posteval_stack.pop().unwrap(); let peek: Option<&T::Ptr> = self.val_stack.last(); - f(peek); + if let Some(peek) = peek { + f(self.allocator, peek); + } 0 } }; @@ -370,15 +373,15 @@ impl<'a, T: Allocator> RunProgramContext<'a, T> { } #[allow(clippy::too_many_arguments)] -pub fn run_program( - allocator: &mut T, +pub fn run_program<'a, T: Allocator>( + allocator: &'a mut T, program: &T::Ptr, args: &T::Ptr, - quote_kw: &[u8], - apply_kw: &[u8], + quote_kw: &'a [u8], + apply_kw: &'a [u8], max_cost: Cost, - operator_lookup: &dyn OperatorHandler, - pre_eval: Option>, + operator_lookup: &'a dyn OperatorHandler, + pre_eval: Option>, ) -> Response { let mut rpc = RunProgramContext::new(allocator, quote_kw, apply_kw, operator_lookup, pre_eval); rpc.run_program(program, args, max_cost) From 8babcbe20ae5b54967a515532729d2d2ca7e2927 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 18 May 2021 15:03:12 -0700 Subject: [PATCH 68/76] Works!! --- src/py/dialect.rs | 63 +++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/src/py/dialect.rs b/src/py/dialect.rs index e43871d4..655763ef 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -1,5 +1,6 @@ use std::cell::RefMut; use std::collections::HashMap; +use std::sync::Arc; use pyo3::prelude::{pyclass, pymethods}; @@ -12,7 +13,7 @@ use crate::int_allocator::IntAllocator; use crate::reduction::EvalErr; use crate::reduction::Reduction; use crate::reduction::Response; -use crate::run_program::{OperatorHandler, PreEval}; +use crate::run_program::{OperatorHandler, PostEval, PreEval}; use crate::serialize::node_from_bytes; use super::error_bridge::{eval_err_for_pyerr, raise_eval_error, unwrap_or_eval_err}; @@ -175,7 +176,7 @@ impl Dialect { max_cost: Cost, pre_eval: &'p PyAny, ) -> PyResult<(Cost, &'p PyAny)> { - let borrowed_arena = arena.borrow(); + let borrowed_arena = Arc::new(arena.borrow()); let mut allocator_refcell: RefMut = borrowed_arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; @@ -185,38 +186,36 @@ impl Dialect { }; let pre_eval_obj = pre_eval.to_object(py); + let pre_eval_f = { + if pre_eval_obj.is_none(py) { + None + } else { + let local_pre_eval: PreEval = Box::new(|allocator, program, args| { + { + // TODO: fix unwraps + let program_obj = borrowed_arena + .py_for_native(py, program, allocator) + .unwrap(); + let args_obj = borrowed_arena.py_for_native(py, args, allocator).unwrap(); + let post_eval_obj = pre_eval_obj + .call1(py, (program_obj, args_obj)) + .unwrap() + .to_object(py); - let local_pre_eval: PreEval = Box::new(|allocator, program, args| { - { - // TODO: fix unwraps - let program_obj = borrowed_arena - .py_for_native(py, program, allocator) - .unwrap(); - let args_obj = borrowed_arena.py_for_native(py, args, allocator).unwrap(); - if !pre_eval_obj.is_none(py) { - let post_eval_obj = pre_eval_obj - .call1(py, (program_obj, args_obj)) - .unwrap() - .to_object(py); - } - - let local_arena = arena.to_object(py); - /* - let post_eval: Box> = - Box::new(move |allocator: &mut IntAllocator, result_ptr: &i32| { - dbg!("** 1"); - let arena: PyRef = local_arena.extract(py).unwrap(); - dbg!("** 1"); - let r = arena.py_for_native(py, &result_ptr, allocator).unwrap(); - dbg!("** 1"); - post_eval_obj.call1(py, (r,)).unwrap(); - dbg!("** 1"); - }); - */ - Ok(None) + let local_borrowed = borrowed_arena.clone(); + let post_eval: Box> = + Box::new(move |allocator: &mut IntAllocator, result_ptr: &i32| { + let r = local_borrowed + .py_for_native(py, &result_ptr, allocator) + .unwrap(); + post_eval_obj.call1(py, (r,)); + }); + Ok(Some(post_eval)) + } + }); + Some(local_pre_eval) } - }); - let pre_eval_f = Some(local_pre_eval); + }; let r: Result, EvalErr> = crate::run_program::run_program( allocator, From f3ac35a4916a197d2be59cb8f9f743507a73eb82 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 18 May 2021 19:01:42 -0700 Subject: [PATCH 69/76] Tidy up a bit. --- src/py/dialect.rs | 55 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 655763ef..acc14631 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -166,6 +166,23 @@ impl Dialect { } } +fn pre_eval_callback( + py: Python, + arena: &PyArena, + pre_eval_obj: PyObject, + allocator: &mut IntAllocator, + program: &i32, + args: &i32, +) -> PyResult { + // call the python `pre_eval` object and return the python object yielded + let program_obj = arena.py_for_native(py, program, allocator)?; + let args_obj = arena.py_for_native(py, args, allocator)?; + let post_eval_obj = pre_eval_obj + .call1(py, (program_obj, args_obj))? + .to_object(py); + Ok(post_eval_obj) +} + impl Dialect { pub fn run_program_ptr<'p>( &self, @@ -176,7 +193,7 @@ impl Dialect { max_cost: Cost, pre_eval: &'p PyAny, ) -> PyResult<(Cost, &'p PyAny)> { - let borrowed_arena = Arc::new(arena.borrow()); + let borrowed_arena: Arc> = Arc::new(arena.borrow()); let mut allocator_refcell: RefMut = borrowed_arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; @@ -185,32 +202,36 @@ impl Dialect { arena: &arena, }; + // we convert `pre_eval` from a python object to a `PreEval` + // this should be factored out to a standalone function, but + // lifetimes make it tough! let pre_eval_obj = pre_eval.to_object(py); let pre_eval_f = { - if pre_eval_obj.is_none(py) { + if pre_eval.is_none() { None } else { let local_pre_eval: PreEval = Box::new(|allocator, program, args| { - { - // TODO: fix unwraps - let program_obj = borrowed_arena - .py_for_native(py, program, allocator) - .unwrap(); - let args_obj = borrowed_arena.py_for_native(py, args, allocator).unwrap(); - let post_eval_obj = pre_eval_obj - .call1(py, (program_obj, args_obj)) - .unwrap() - .to_object(py); - + if let Ok(post_eval_obj) = pre_eval_callback( + py, + &borrowed_arena, + pre_eval_obj.clone(), + allocator, + program, + args, + ) { let local_borrowed = borrowed_arena.clone(); let post_eval: Box> = Box::new(move |allocator: &mut IntAllocator, result_ptr: &i32| { - let r = local_borrowed - .py_for_native(py, &result_ptr, allocator) - .unwrap(); - post_eval_obj.call1(py, (r,)); + if let Ok(r) = + local_borrowed.py_for_native(py, &result_ptr, allocator) + { + // invoke the python `PostEval` callback + let _r = post_eval_obj.call1(py, (r,)); + } }); Ok(Some(post_eval)) + } else { + Ok(None) } }); Some(local_pre_eval) From 636c22668c4fa59966e47cfd1b051ca7119302ac Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 18 May 2021 19:52:17 -0700 Subject: [PATCH 70/76] Get rid of `PyView`. --- src/py/mod.rs | 1 - src/py/py_arena.rs | 25 +++++++++++++++---------- src/py/py_view.rs | 19 ------------------- 3 files changed, 15 insertions(+), 30 deletions(-) delete mode 100644 src/py/py_view.rs diff --git a/src/py/mod.rs b/src/py/mod.rs index 8e5b1e89..6a5ff197 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -4,5 +4,4 @@ pub mod error_bridge; pub mod f_table; pub mod native_op; pub mod py_arena; -pub mod py_view; pub mod run_program; diff --git a/src/py/py_arena.rs b/src/py/py_arena.rs index dc41b403..71026f56 100644 --- a/src/py/py_arena.rs +++ b/src/py/py_arena.rs @@ -9,8 +9,6 @@ use crate::allocator::{Allocator, SExp}; use crate::int_allocator::IntAllocator; use crate::serialize::node_from_bytes; -use super::py_view::PyView; - #[pyclass(subclass, unsendable)] pub struct PyArena { arena: RefCell, @@ -18,6 +16,17 @@ pub struct PyArena { to_python: PyObject, } +pub fn sexp_for_obj(obj: &PyAny) -> PyResult> { + let r: PyResult<&PyBytes> = obj.getattr("atom")?.extract(); + if let Ok(bytes) = r { + return Ok(SExp::Atom(bytes)); + } + let pair: &PyTuple = obj.getattr("pair")?.extract()?; + let p0: &PyAny = pair.get_item(0); + let p1: &PyAny = pair.get_item(1); + Ok(SExp::Pair(p0, p1)) +} + #[pymethods] impl PyArena { #[new] @@ -110,19 +119,15 @@ impl PyArena { // it's not in the cache - match PyView::py_view_for_obj(obj)? { - PyView::Atom(atom) => { - let blob: &[u8] = atom.extract(py).unwrap(); + match sexp_for_obj(obj)? { + SExp::Atom(atom) => { + let blob: &[u8] = atom.extract()?; let ptr = allocator.new_atom(blob).unwrap(); self.add(py, obj, &ptr)?; Ok(None) } - PyView::Pair(pair) => { - let pair: &PyAny = pair.into_ref(py); - let pair: &PyTuple = pair.extract()?; - let p0: &PyAny = pair.get_item(0); - let p1: &PyAny = pair.get_item(1); + SExp::Pair(p0, p1) => { let ptr_0: PyResult = self.from_py_to_native_cache(py, p0); let ptr_1: PyResult = self.from_py_to_native_cache(py, p1); diff --git a/src/py/py_view.rs b/src/py/py_view.rs deleted file mode 100644 index a86ec958..00000000 --- a/src/py/py_view.rs +++ /dev/null @@ -1,19 +0,0 @@ -use pyo3::{PyAny, PyObject, PyResult}; - -#[derive(Clone)] -pub enum PyView { - Atom(PyObject), - Pair(PyObject), -} - -impl PyView { - // TODO: move this into `py_arena` and use very similar `SExp<'p, &'p PyBytes, &'p PyAny>` instead - pub fn py_view_for_obj(obj: &PyAny) -> PyResult { - let r: &PyAny = obj.getattr("atom")?.extract()?; - if !r.is_none() { - return Ok(PyView::Atom(r.into())); - } - let r: &PyAny = obj.getattr("pair")?.extract()?; - Ok(PyView::Pair(r.into())) - } -} From 1f2ac7e324122934825ed48e9a9f089e514576c1 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 18 May 2021 22:23:43 -0700 Subject: [PATCH 71/76] Rename `PyArena` to `Arena`. --- src/py/api.rs | 8 ++++---- src/py/{py_arena.rs => arena.rs} | 12 ++++++------ src/py/dialect.rs | 22 +++++++++++----------- src/py/error_bridge.rs | 6 +++--- src/py/mod.rs | 2 +- src/py/native_op.rs | 8 +++----- src/py/run_program.rs | 4 ++-- 7 files changed, 30 insertions(+), 32 deletions(-) rename src/py/{py_arena.rs => arena.rs} (98%) diff --git a/src/py/api.rs b/src/py/api.rs index e78fde09..9e0edb42 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -12,10 +12,10 @@ use crate::more_ops::*; use crate::node::Node; use crate::serialize::node_to_bytes; +use super::arena::Arena; use super::dialect::{Dialect, PyMultiOpFn}; use super::f_table::OpFn; use super::native_op::NativeOp; -use super::py_arena::PyArena; use super::run_program::{ __pyo3_get_function_deserialize_and_run_program, __pyo3_get_function_serialized_length, }; @@ -63,12 +63,12 @@ pub fn native_opcodes_dict(py: Python) -> PyResult { #[pyfunction] fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyAny) -> PyResult<&'p PyBytes> { - let arena = PyArena::new_cell(py)?; + let arena = Arena::new_cell(py)?; let arena_borrowed = arena.borrow(); let mut allocator_refcell: RefMut = arena_borrowed.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let ptr = PyArena::native_for_py(arena, py, sexp, allocator)?; + let ptr = Arena::native_for_py(arena, py, sexp, allocator)?; let node = Node::new(allocator, ptr); let s: Vec = node_to_bytes(&node)?; @@ -78,8 +78,8 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyAny) -> PyResult<&'p PyBytes> /// This module is a python module implemented in Rust. #[pymodule] fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_class::()?; m.add_class::()?; - m.add_class::()?; m.add_function(wrap_pyfunction!(native_opcodes_dict, m)?)?; diff --git a/src/py/py_arena.rs b/src/py/arena.rs similarity index 98% rename from src/py/py_arena.rs rename to src/py/arena.rs index 71026f56..2ecf9ab7 100644 --- a/src/py/py_arena.rs +++ b/src/py/arena.rs @@ -10,7 +10,7 @@ use crate::int_allocator::IntAllocator; use crate::serialize::node_from_bytes; #[pyclass(subclass, unsendable)] -pub struct PyArena { +pub struct Arena { arena: RefCell, cache: PyObject, to_python: PyObject, @@ -28,10 +28,10 @@ pub fn sexp_for_obj(obj: &PyAny) -> PyResult> { } #[pymethods] -impl PyArena { +impl Arena { #[new] pub fn new(py: Python, new_obj_f: PyObject) -> PyResult { - Ok(PyArena { + Ok(Arena { arena: RefCell::new(IntAllocator::default()), cache: py.eval("dict()", None, None)?.to_object(py), to_python: new_obj_f, @@ -55,9 +55,9 @@ impl PyArena { } } -impl PyArena { +impl Arena { pub fn new_cell_obj(py: Python, new_obj_f: PyObject) -> PyResult<&PyCell> { - PyCell::new(py, PyArena::new(py, new_obj_f)?) + PyCell::new(py, Arena::new(py, new_obj_f)?) } pub fn new_cell(py: Python) -> PyResult<&PyCell> { @@ -161,7 +161,7 @@ impl PyArena { } pub fn native_for_py( - slf: &PyCell, + slf: &PyCell, py: Python, obj: &PyAny, allocator: &mut IntAllocator, diff --git a/src/py/dialect.rs b/src/py/dialect.rs index acc14631..2757ca7c 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -16,11 +16,11 @@ use crate::reduction::Response; use crate::run_program::{OperatorHandler, PostEval, PreEval}; use crate::serialize::node_from_bytes; +use super::arena::Arena; use super::error_bridge::{eval_err_for_pyerr, raise_eval_error, unwrap_or_eval_err}; use super::f_table::FLookup; use super::f_table::OpFn; use super::native_op::NativeOp; -use super::py_arena::PyArena; #[pyclass] #[derive(Clone)] @@ -134,8 +134,8 @@ impl Dialect { max_cost: Cost, pre_eval_f: &PyAny, ) -> PyResult<(Cost, PyObject)> { - let arena = PyArena::new_cell_obj(py, self.to_python.clone())?; - let arena_ptr: &PyArena = &arena.borrow() as &PyArena; + let arena = Arena::new_cell_obj(py, self.to_python.clone())?; + let arena_ptr: &Arena = &arena.borrow() as &Arena; let program = arena_ptr.ptr_for_obj(py, program)?; let args = arena_ptr.ptr_for_obj(py, args)?; @@ -152,9 +152,9 @@ impl Dialect { max_cost: Cost, pre_eval: &'p PyAny, ) -> PyResult<(Cost, &'p PyAny)> { - let arena = PyArena::new_cell_obj(py, self.to_python.clone())?; + let arena = Arena::new_cell_obj(py, self.to_python.clone())?; let (program, args) = { - let arena_ptr: &PyArena = &arena.borrow() as &PyArena; + let arena_ptr: &Arena = &arena.borrow() as &Arena; let mut allocator_refcell: RefMut = arena_ptr.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; @@ -168,7 +168,7 @@ impl Dialect { fn pre_eval_callback( py: Python, - arena: &PyArena, + arena: &Arena, pre_eval_obj: PyObject, allocator: &mut IntAllocator, program: &i32, @@ -187,13 +187,13 @@ impl Dialect { pub fn run_program_ptr<'p>( &self, py: Python<'p>, - arena: &'p PyCell, + arena: &'p PyCell, program: i32, args: i32, max_cost: Cost, pre_eval: &'p PyAny, ) -> PyResult<(Cost, &'p PyAny)> { - let borrowed_arena: Arc> = Arc::new(arena.borrow()); + let borrowed_arena: Arc> = Arc::new(arena.borrow()); let mut allocator_refcell: RefMut = borrowed_arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; @@ -272,7 +272,7 @@ impl Dialect { struct DialectRunningContext<'a> { dialect: &'a Dialect, - arena: &'a PyCell, + arena: &'a PyCell, } impl DialectRunningContext<'_> { @@ -285,7 +285,7 @@ impl DialectRunningContext<'_> { ) -> Response<::Ptr> { Python::with_gil(|py| { let r = unwrap_or_eval_err( - PyArena::py_for_native(&self.arena.borrow(), py, args, allocator), + Arena::py_for_native(&self.arena.borrow(), py, args, allocator), args, "can't uncache", )?; @@ -307,7 +307,7 @@ impl DialectRunningContext<'_> { let clvm_object: &PyAny = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = PyArena::native_for_py(self.arena, py, clvm_object, allocator); + let r = Arena::native_for_py(self.arena, py, clvm_object, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } diff --git a/src/py/error_bridge.rs b/src/py/error_bridge.rs index dd233d95..77168c0d 100644 --- a/src/py/error_bridge.rs +++ b/src/py/error_bridge.rs @@ -3,20 +3,20 @@ use crate::reduction::EvalErr; use pyo3::types::{PyDict, PyString, PyTuple}; use pyo3::{PyAny, PyCell, PyErr, PyObject, PyResult, Python}; -use super::py_arena::PyArena; +use super::arena::Arena; /// turn a `PyErr` into an `EvalErr

` if at all possible /// otherwise, return a `PyErr` pub fn eval_err_for_pyerr<'p>( py: Python<'p>, pyerr: &PyErr, - arena: &'p PyCell, + arena: &'p PyCell, allocator: &mut IntAllocator, ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; let sexp: &PyAny = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = PyArena::native_for_py(arena, py, sexp, allocator)?; + let node: i32 = Arena::native_for_py(arena, py, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) } diff --git a/src/py/mod.rs b/src/py/mod.rs index 6a5ff197..a86d640f 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -1,7 +1,7 @@ pub mod api; +pub mod arena; pub mod dialect; pub mod error_bridge; pub mod f_table; pub mod native_op; -pub mod py_arena; pub mod run_program; diff --git a/src/py/native_op.rs b/src/py/native_op.rs index 6caeddf0..5932e083 100644 --- a/src/py/native_op.rs +++ b/src/py/native_op.rs @@ -6,11 +6,9 @@ use crate::cost::Cost; use crate::int_allocator::IntAllocator; use crate::reduction::Reduction; +use super::arena::Arena; use super::error_bridge::raise_eval_error; use super::f_table::OpFn; -use super::py_arena::PyArena; - -//type OpFn = fn(&mut T, ::Ptr, Cost) -> Response<::Ptr>; #[pyclass] pub struct NativeOp { @@ -32,8 +30,8 @@ impl NativeOp { args: &'p PyAny, _max_cost: Cost, ) -> PyResult<(Cost, PyObject)> { - let arena_cell = PyArena::new_cell(py)?; - let arena: &PyArena = &arena_cell.borrow(); + let arena_cell = Arena::new_cell(py)?; + let arena: &Arena = &arena_cell.borrow(); let ptr = arena.ptr_for_obj(py, args)?; let mut allocator = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator; diff --git a/src/py/run_program.rs b/src/py/run_program.rs index b709dac3..4fd82bd4 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -11,8 +11,8 @@ use crate::reduction::Response; use crate::run_program::{run_program, OperatorHandler}; use crate::serialize::{node_from_bytes, node_to_bytes, serialized_length_from_bytes}; +use super::arena::Arena; use super::f_table::{f_lookup_for_hashmap, FLookup}; -use super::py_arena::PyArena; use pyo3::prelude::*; use pyo3::types::{PyBytes, PyDict}; @@ -58,7 +58,7 @@ pub fn deserialize_and_run_program( max_cost: Cost, flags: u32, ) -> PyResult<(Cost, PyObject)> { - let arena = PyArena::new_cell(py)?; + let arena = Arena::new_cell(py)?; let arena_borrowed = arena.borrow(); let mut allocator_refcell: RefMut = arena_borrowed.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; From 00df1af3ef97af75b8ca385f91ab698e4e3eb4bb Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 18 May 2021 22:53:46 -0700 Subject: [PATCH 72/76] Add comments. --- src/py/api.rs | 1 - src/py/arena.rs | 21 +++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/py/api.rs b/src/py/api.rs index 9e0edb42..e3cf4fed 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -82,7 +82,6 @@ fn clvm_rs(_py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_function(wrap_pyfunction!(native_opcodes_dict, m)?)?; - m.add_function(wrap_pyfunction!(serialized_length, m)?)?; m.add( diff --git a/src/py/arena.rs b/src/py/arena.rs index 2ecf9ab7..93aff3f7 100644 --- a/src/py/arena.rs +++ b/src/py/arena.rs @@ -1,3 +1,7 @@ +/// An `Arena` is a collection of objects representing a program and +/// its arguments, and intermediate values reached while running +/// a program. Objects can be created in an `Arena` but are never +/// dropped until the `Arena` is dropped. use std::cell::{RefCell, RefMut}; use std::collections::HashSet; @@ -12,10 +16,20 @@ use crate::serialize::node_from_bytes; #[pyclass(subclass, unsendable)] pub struct Arena { arena: RefCell, + /// this cache is a python `dict` that keeps a mapping + /// from `i32` to python objects and vice-versa cache: PyObject, + /// `to_python`, a python callable, is called whenever a `python` + /// object needs to be created from a native one. It's called with + /// either a `bytes` or `tuple` of two elements, each of which + /// either came from python or has had `to_python` called on them to_python: PyObject, } +/// yield a corresponding `SExp` for a python object based +/// on the standard way of looking in `.atom` for a `PyBytes` object +/// and then in `.pair` for a `PyTuple`. + pub fn sexp_for_obj(obj: &PyAny) -> PyResult> { let r: PyResult<&PyBytes> = obj.getattr("atom")?.extract(); if let Ok(bytes) = r { @@ -38,17 +52,22 @@ impl Arena { }) } + /// deserialize `bytes` into an object in this `Arena` pub fn deserialize<'p>(&self, py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyAny> { let allocator: &mut IntAllocator = &mut self.allocator() as &mut IntAllocator; let ptr = node_from_bytes(allocator, blob)?; self.py_for_native(py, &ptr, allocator) } + /// copy this python object into this `Arena` if it's not yet in the cache + /// (otherwise it returns the previously cached object) pub fn include<'p>(&self, py: Python<'p>, obj: &'p PyAny) -> PyResult<&'p PyAny> { let ptr = Self::ptr_for_obj(self, py, obj)?; self.py_for_native(py, &ptr, &mut self.allocator() as &mut IntAllocator) } + /// copy this python object into this `Arena` if it's not yet in the cache + /// (otherwise it returns the previously cached object) pub fn ptr_for_obj(&self, py: Python, obj: &PyAny) -> PyResult { let allocator: &mut IntAllocator = &mut self.allocator() as &mut IntAllocator; self.populate_native(py, obj, allocator) @@ -72,6 +91,8 @@ impl Arena { self.arena.borrow_mut() } + /// add a python object <-> native object mapping + /// to the cache, in both directions pub fn add( &self, py: Python, From ede6ca9f9fc20f8eb481d9639c4099e7fe5d7128 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Sat, 22 May 2021 14:39:53 -0700 Subject: [PATCH 73/76] Prefer `PyRef` over `PyCell`. --- src/py/api.rs | 8 +++---- src/py/arena.rs | 7 +++--- src/py/dialect.rs | 51 ++++++++++++++++++++---------------------- src/py/error_bridge.rs | 4 ++-- 4 files changed, 33 insertions(+), 37 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index e3cf4fed..400e196f 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -63,12 +63,12 @@ pub fn native_opcodes_dict(py: Python) -> PyResult { #[pyfunction] fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyAny) -> PyResult<&'p PyBytes> { - let arena = Arena::new_cell(py)?; - let arena_borrowed = arena.borrow(); - let mut allocator_refcell: RefMut = arena_borrowed.allocator(); + let arena_cell = Arena::new_cell(py)?; + let arena = arena_cell.borrow(); + let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; - let ptr = Arena::native_for_py(arena, py, sexp, allocator)?; + let ptr = Arena::native_for_py(&arena, py, sexp, allocator)?; let node = Node::new(allocator, ptr); let s: Vec = node_to_bytes(&node)?; diff --git a/src/py/arena.rs b/src/py/arena.rs index 93aff3f7..f3288e37 100644 --- a/src/py/arena.rs +++ b/src/py/arena.rs @@ -182,14 +182,13 @@ impl Arena { } pub fn native_for_py( - slf: &PyCell, + slf: &PyRef, py: Python, obj: &PyAny, allocator: &mut IntAllocator, ) -> PyResult<::Ptr> { - let celf = slf.borrow(); - celf.from_py_to_native_cache(py, obj) - .or_else(|_err| celf.populate_native(py, obj, allocator)) + slf.from_py_to_native_cache(py, obj) + .or_else(|_err| slf.populate_native(py, obj, allocator)) } // native to py methods diff --git a/src/py/dialect.rs b/src/py/dialect.rs index 2757ca7c..c4993df7 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -1,11 +1,10 @@ use std::cell::RefMut; use std::collections::HashMap; -use std::sync::Arc; use pyo3::prelude::{pyclass, pymethods}; use pyo3::types::{PyString, PyTuple}; -use pyo3::{FromPyObject, PyAny, PyCell, PyObject, PyRef, PyResult, Python, ToPyObject}; +use pyo3::{FromPyObject, PyAny, PyObject, PyRef, PyResult, Python, ToPyObject}; use crate::allocator::Allocator; use crate::cost::Cost; @@ -66,9 +65,9 @@ impl MultiOpFnE { impl<'source> FromPyObject<'source> for MultiOpFnE { fn extract(obj: &'source pyo3::PyAny) -> PyResult { - let v: PyResult<&PyCell> = obj.extract(); + let v: PyResult> = obj.extract(); if let Ok(v) = v { - Ok(Self::Rust(v.borrow().op)) + Ok(Self::Rust(v.op)) } else { Ok(Self::Python(obj.into())) } @@ -134,13 +133,13 @@ impl Dialect { max_cost: Cost, pre_eval_f: &PyAny, ) -> PyResult<(Cost, PyObject)> { - let arena = Arena::new_cell_obj(py, self.to_python.clone())?; - let arena_ptr: &Arena = &arena.borrow() as &Arena; + let arena_cell = Arena::new_cell_obj(py, self.to_python.clone())?; + let arena: PyRef = arena_cell.borrow(); - let program = arena_ptr.ptr_for_obj(py, program)?; - let args = arena_ptr.ptr_for_obj(py, args)?; + let program = arena.ptr_for_obj(py, program)?; + let args = arena.ptr_for_obj(py, args)?; - let (cost, r) = self.run_program_ptr(py, &arena, program, args, max_cost, pre_eval_f)?; + let (cost, r) = self.run_program_ptr(py, arena, program, args, max_cost, pre_eval_f)?; Ok((cost, r.to_object(py))) } @@ -152,17 +151,17 @@ impl Dialect { max_cost: Cost, pre_eval: &'p PyAny, ) -> PyResult<(Cost, &'p PyAny)> { - let arena = Arena::new_cell_obj(py, self.to_python.clone())?; + let arena_cell = Arena::new_cell_obj(py, self.to_python.clone())?; + let arena = arena_cell.borrow(); let (program, args) = { - let arena_ptr: &Arena = &arena.borrow() as &Arena; - let mut allocator_refcell: RefMut = arena_ptr.allocator(); + let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; let program = node_from_bytes(allocator, program_blob)?; let args = node_from_bytes(allocator, args_blob)?; (program, args) }; - self.run_program_ptr(py, &arena, program, args, max_cost, pre_eval) + self.run_program_ptr(py, arena, program, args, max_cost, pre_eval) } } @@ -187,14 +186,13 @@ impl Dialect { pub fn run_program_ptr<'p>( &self, py: Python<'p>, - arena: &'p PyCell, + arena: PyRef<'p, Arena>, program: i32, args: i32, max_cost: Cost, pre_eval: &'p PyAny, ) -> PyResult<(Cost, &'p PyAny)> { - let borrowed_arena: Arc> = Arc::new(arena.borrow()); - let mut allocator_refcell: RefMut = borrowed_arena.allocator(); + let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut IntAllocator = &mut allocator_refcell as &mut IntAllocator; let drc = DialectRunningContext { @@ -213,20 +211,19 @@ impl Dialect { let local_pre_eval: PreEval = Box::new(|allocator, program, args| { if let Ok(post_eval_obj) = pre_eval_callback( py, - &borrowed_arena, + &arena, pre_eval_obj.clone(), allocator, program, args, ) { - let local_borrowed = borrowed_arena.clone(); + let local_arena = &arena; let post_eval: Box> = Box::new(move |allocator: &mut IntAllocator, result_ptr: &i32| { - if let Ok(r) = - local_borrowed.py_for_native(py, &result_ptr, allocator) + if let Ok(r) = local_arena.py_for_native(py, &result_ptr, allocator) { // invoke the python `PostEval` callback - let _r = post_eval_obj.call1(py, (r,)); + let _r = post_eval_obj.call1(py, (r.to_object(py),)); } }); Ok(Some(post_eval)) @@ -251,11 +248,11 @@ impl Dialect { match r { Ok(reduction) => { - let r = borrowed_arena.py_for_native(py, &reduction.1, allocator)?; + let r = arena.py_for_native(py, &reduction.1, allocator)?; Ok((reduction.0, r)) } Err(eval_err) => { - let node: PyObject = borrowed_arena + let node: PyObject = arena .py_for_native(py, &eval_err.0, allocator)? .to_object(py); let s: String = eval_err.1; @@ -272,7 +269,7 @@ impl Dialect { struct DialectRunningContext<'a> { dialect: &'a Dialect, - arena: &'a PyCell, + arena: &'a PyRef<'a, Arena>, } impl DialectRunningContext<'_> { @@ -285,7 +282,7 @@ impl DialectRunningContext<'_> { ) -> Response<::Ptr> { Python::with_gil(|py| { let r = unwrap_or_eval_err( - Arena::py_for_native(&self.arena.borrow(), py, args, allocator), + Arena::py_for_native(&self.arena, py, args, allocator), args, "can't uncache", )?; @@ -293,7 +290,7 @@ impl DialectRunningContext<'_> { match r1 { Err(pyerr) => { let eval_err: PyResult> = - eval_err_for_pyerr(py, &pyerr, self.arena, allocator); + eval_err_for_pyerr(py, &pyerr, &self.arena, allocator); let r: EvalErr = unwrap_or_eval_err(eval_err, args, "unexpected exception")?; Err(r) @@ -307,7 +304,7 @@ impl DialectRunningContext<'_> { let clvm_object: &PyAny = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = Arena::native_for_py(self.arena, py, clvm_object, allocator); + let r = Arena::native_for_py(&self.arena, py, clvm_object, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } diff --git a/src/py/error_bridge.rs b/src/py/error_bridge.rs index 77168c0d..41810f73 100644 --- a/src/py/error_bridge.rs +++ b/src/py/error_bridge.rs @@ -1,7 +1,7 @@ use crate::int_allocator::IntAllocator; use crate::reduction::EvalErr; use pyo3::types::{PyDict, PyString, PyTuple}; -use pyo3::{PyAny, PyCell, PyErr, PyObject, PyResult, Python}; +use pyo3::{PyAny, PyErr, PyObject, PyRef, PyResult, Python}; use super::arena::Arena; @@ -10,7 +10,7 @@ use super::arena::Arena; pub fn eval_err_for_pyerr<'p>( py: Python<'p>, pyerr: &PyErr, - arena: &'p PyCell, + arena: &'p PyRef, allocator: &mut IntAllocator, ) -> PyResult> { let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; From e4223c916a7b8096183528aa00c1079c7014c5f2 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 30 Jun 2021 20:34:56 -0700 Subject: [PATCH 74/76] Factor out `BridgeCache`. --- src/py/api.rs | 2 +- src/py/arena.rs | 235 ++------------------------------------ src/py/bridge_cache.rs | 248 +++++++++++++++++++++++++++++++++++++++++ src/py/dialect.rs | 18 +-- src/py/error_bridge.rs | 2 +- src/py/mod.rs | 1 + src/py/native_op.rs | 4 +- src/py/run_program.rs | 1 + 8 files changed, 274 insertions(+), 237 deletions(-) create mode 100644 src/py/bridge_cache.rs diff --git a/src/py/api.rs b/src/py/api.rs index 948aead7..33f5cb0f 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -70,7 +70,7 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyAny) -> PyResult<&'p PyBytes> let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut Allocator = &mut allocator_refcell as &mut Allocator; - let ptr = Arena::native_for_py(&arena, py, sexp, allocator)?; + let ptr = arena.cache.native_for_py(py, sexp, allocator)?; let node = Node::new(allocator, ptr); let s: Vec = node_to_bytes(&node)?; diff --git a/src/py/arena.rs b/src/py/arena.rs index ef5a8672..286f0d51 100644 --- a/src/py/arena.rs +++ b/src/py/arena.rs @@ -3,46 +3,19 @@ /// a program. Objects can be created in an `Arena` but are never /// dropped until the `Arena` is dropped. use std::cell::{RefCell, RefMut}; -use std::collections::HashSet; use pyo3::prelude::pyclass; use pyo3::prelude::*; -use pyo3::types::{IntoPyDict, PyBytes, PyTuple}; -use crate::allocator::{Allocator, NodePtr, SExp}; +use super::bridge_cache::BridgeCache; +use crate::allocator::Allocator; use crate::serialize::node_from_bytes; -pub enum PySExp<'p> { - Atom(&'p PyBytes), - Pair(&'p PyAny, &'p PyAny), -} - #[pyclass(subclass, unsendable)] pub struct Arena { arena: RefCell, - /// this cache is a python `dict` that keeps a mapping - /// from `i32` to python objects and vice-versa - cache: PyObject, - /// `to_python`, a python callable, is called whenever a `python` - /// object needs to be created from a native one. It's called with - /// either a `bytes` or `tuple` of two elements, each of which - /// either came from python or has had `to_python` called on them - to_python: PyObject, -} - -/// yield a corresponding `PySExp` for a python object based -/// on the standard way of looking in `.atom` for a `PyBytes` object -/// and then in `.pair` for a `PyTuple`. - -pub fn sexp_for_obj(obj: &PyAny) -> PyResult { - let r: PyResult<&PyBytes> = obj.getattr("atom")?.extract(); - if let Ok(bytes) = r { - return Ok(PySExp::Atom(bytes)); - } - let pair: &PyTuple = obj.getattr("pair")?.extract()?; - let p0: &PyAny = pair.get_item(0); - let p1: &PyAny = pair.get_item(1); - Ok(PySExp::Pair(p0, p1)) + pub cache: BridgeCache, + // TODO: make this private, remove this `pub` } #[pymethods] @@ -51,8 +24,7 @@ impl Arena { pub fn new(py: Python, new_obj_f: PyObject) -> PyResult { Ok(Arena { arena: RefCell::new(Allocator::default()), - cache: py.eval("dict()", None, None)?.to_object(py), - to_python: new_obj_f, + cache: BridgeCache::new(py, new_obj_f)?, }) } @@ -60,21 +32,22 @@ impl Arena { pub fn deserialize<'p>(&self, py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyAny> { let allocator: &mut Allocator = &mut self.allocator() as &mut Allocator; let ptr = node_from_bytes(allocator, blob)?; - self.py_for_native(py, ptr, allocator) + self.cache.py_for_native(py, ptr, allocator) } /// copy this python object into this `Arena` if it's not yet in the cache /// (otherwise it returns the previously cached object) pub fn include<'p>(&self, py: Python<'p>, obj: &'p PyAny) -> PyResult<&'p PyAny> { let ptr = Self::ptr_for_obj(self, py, obj)?; - self.py_for_native(py, ptr, &mut self.allocator() as &mut Allocator) + self.cache + .py_for_native(py, ptr, &mut self.allocator() as &mut Allocator) } /// copy this python object into this `Arena` if it's not yet in the cache /// (otherwise it returns the previously cached object) pub fn ptr_for_obj(&self, py: Python, obj: &PyAny) -> PyResult { let allocator: &mut Allocator = &mut self.allocator() as &mut Allocator; - self.populate_native(py, obj, allocator) + self.cache.populate_native(py, obj, allocator) } } @@ -88,198 +61,10 @@ impl Arena { } pub fn obj_for_ptr<'p>(&self, py: Python<'p>, ptr: i32) -> PyResult<&'p PyAny> { - self.py_for_native(py, ptr, &mut self.allocator()) + self.cache.py_for_native(py, ptr, &mut self.allocator()) } pub fn allocator(&self) -> RefMut { self.arena.borrow_mut() } - - /// add a python object <-> native object mapping - /// to the cache, in both directions - pub fn add(&self, py: Python, obj: &PyAny, ptr: NodePtr) -> PyResult<()> { - let locals = [ - ("cache", self.cache.clone()), - ("obj", obj.to_object(py)), - ("ptr", ptr.to_object(py)), - ] - .into_py_dict(py); - - py.run("cache[ptr] = obj; cache[id(obj)] = ptr", None, Some(locals)) - } - - // py to native methods - - fn from_py_to_native_cache<'p>(&self, py: Python<'p>, obj: &PyAny) -> PyResult { - let locals = [("cache", self.cache.clone()), ("key", obj.to_object(py))].into_py_dict(py); - py.eval("cache.get(id(key))", None, Some(locals))?.extract() - } - - fn populate_native( - &self, - py: Python, - obj: &PyAny, - allocator: &mut Allocator, - ) -> PyResult { - // items in `pending` are already in the stack of things to be converted - // if they appear again, we have an illegal cycle and must fail - - let mut pending: HashSet = HashSet::new(); - - apply_to_tree(obj, move |obj| { - // is it in cache yet? - if self.from_py_to_native_cache(py, obj).is_ok() { - // yep, we're done - return Ok(None); - } - - // it's not in the cache - - match sexp_for_obj(obj)? { - PySExp::Atom(atom) => { - let blob: &[u8] = atom.extract()?; - let ptr = allocator.new_atom(blob).unwrap(); - self.add(py, obj, ptr)?; - - Ok(None) - } - PySExp::Pair(p0, p1) => { - let ptr_0: PyResult = self.from_py_to_native_cache(py, p0); - let ptr_1: PyResult = self.from_py_to_native_cache(py, p1); - - let as_obj = id_for_pyany(py, obj)?; - - if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { - let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); - self.add(py, obj, ptr)?; - - pending.remove(&as_obj); - Ok(None) - } else { - if pending.contains(&as_obj) { - let locals = Some([("obj", obj)].into_py_dict(py)); - py.run( - "raise ValueError(f'illegal clvm object loop {obj}')", - None, - locals, - )?; - panic!(); - } - pending.insert(as_obj); - - Ok(Some((p0, p1))) - } - } - } - })?; - - self.from_py_to_native_cache(py, obj) - } - - pub fn native_for_py( - slf: &PyRef, - py: Python, - obj: &PyAny, - allocator: &mut Allocator, - ) -> PyResult { - slf.from_py_to_native_cache(py, obj) - .or_else(|_err| slf.populate_native(py, obj, allocator)) - } - - // native to py methods - - fn from_native_to_py_cache<'p>(&self, py: Python<'p>, ptr: NodePtr) -> PyResult<&'p PyAny> { - let locals = [("cache", self.cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); - py.eval("cache[key]", None, Some(locals))?.extract() - } - - fn populate_python<'p>( - &self, - py: Python<'p>, - ptr: NodePtr, - allocator: &mut Allocator, - ) -> PyResult<&'p PyAny> { - apply_to_tree(ptr, move |ptr| { - // is it in cache yet? - if self.from_native_to_py_cache(py, ptr).is_ok() { - // yep, we're done - return Ok(None); - } - - // it's not in the cache - - match allocator.sexp(ptr) { - SExp::Atom(a) => { - // it's an atom, so we just populate cache directly - let blob = allocator.buf(&a); - let py_bytes = PyBytes::new(py, blob); - self.add(py, self.to_python.as_ref(py).call1((py_bytes,))?, ptr)?; - Ok(None) - } - SExp::Pair(ptr_1, ptr_2) => { - // we can only create this if the children are in the cache - // Let's find out - let locals = [ - ("cache", self.cache.clone()), - ("p1", ptr_1.to_object(py)), - ("p2", ptr_2.to_object(py)), - ] - .into_py_dict(py); - - let pair: PyResult<&PyAny> = - py.eval("(cache[p1], cache[p2])", None, Some(locals)); - - match pair { - // the children aren't in the cache, keep drilling down - Err(_) => Ok(Some((ptr_1, ptr_2))), - - // the children are in the cache, create new node & populate cache with it - Ok(tuple) => { - let (_p1, _p2): (&PyAny, &PyAny) = tuple.extract()?; - self.add(py, self.to_python.as_ref(py).call1((tuple,))?, ptr)?; - Ok(None) - } - } - } - } - })?; - - self.from_native_to_py_cache(py, ptr) - } - - pub fn py_for_native<'p>( - &self, - py: Python<'p>, - ptr: NodePtr, - allocator: &mut Allocator, - ) -> PyResult<&'p PyAny> { - self.from_native_to_py_cache(py, ptr) - .or_else(|_err| self.populate_python(py, ptr, allocator)) - } -} - -fn id_for_pyany(py: Python, obj: &PyAny) -> PyResult { - let locals = Some([("obj", obj)].into_py_dict(py)); - py.eval("id(obj)", None, locals)?.extract() -} - -fn apply_to_tree(node: T, mut apply: F) -> PyResult<()> -where - F: FnMut(T) -> PyResult>, - T: Clone, -{ - let mut items = vec![node]; - loop { - let t = items.pop(); - if let Some(obj) = t { - if let Some((p0, p1)) = apply(obj.clone())? { - items.push(obj); - items.push(p0); - items.push(p1); - } - } else { - break; - } - } - Ok(()) } diff --git a/src/py/bridge_cache.rs b/src/py/bridge_cache.rs new file mode 100644 index 00000000..88b421e0 --- /dev/null +++ b/src/py/bridge_cache.rs @@ -0,0 +1,248 @@ +use std::collections::HashSet; + +use pyo3::prelude::*; +use pyo3::types::{IntoPyDict, PyBytes, PyTuple}; + +use crate::allocator::{Allocator, NodePtr, SExp}; + +enum PySExp<'p> { + Atom(&'p PyBytes), + Pair(&'p PyAny, &'p PyAny), +} + +pub struct BridgeCache { + /// this cache is a python `dict` that keeps a mapping + /// from `i32` to python objects and vice-versa + cache: PyObject, + /// `to_python`, a python callable, is called whenever a `python` + /// object needs to be created from a native one. It's called with + /// either a `bytes` or `tuple` of two elements, each of which + /// either came from python or has had `to_python` called on them + to_python: PyObject, +} + +/// yield a corresponding `PySExp` for a python object based +/// on the standard way of looking in `.atom` for a `PyBytes` object +/// and then in `.pair` for a `PyTuple`. + +fn sexp_for_obj(obj: &PyAny) -> PyResult { + let r: PyResult<&PyBytes> = obj.getattr("atom")?.extract(); + if let Ok(bytes) = r { + return Ok(PySExp::Atom(bytes)); + } + let pair: &PyTuple = obj.getattr("pair")?.extract()?; + let p0: &PyAny = pair.get_item(0); + let p1: &PyAny = pair.get_item(1); + Ok(PySExp::Pair(p0, p1)) +} + +impl BridgeCache { + pub fn new(py: Python, new_obj_f: PyObject) -> PyResult { + Ok(BridgeCache { + cache: py.eval("dict()", None, None)?.to_object(py), + to_python: new_obj_f, + }) + } + + /// copy this python object into this `BridgeCache` if it's not yet in the cache + /// (otherwise it returns the previously cached object) + pub fn include<'p>( + &self, + py: Python<'p>, + allocator: &mut Allocator, + obj: &'p PyAny, + ) -> PyResult<&'p PyAny> { + let ptr = self.populate_native(py, obj, allocator)?; + self.py_for_native(py, ptr, allocator) + } + + /// add a python object <-> native object mapping + /// to the cache, in both directions + pub fn add(&self, py: Python, obj: &PyAny, ptr: NodePtr) -> PyResult<()> { + let locals = [ + ("cache", self.cache.clone()), + ("obj", obj.to_object(py)), + ("ptr", ptr.to_object(py)), + ] + .into_py_dict(py); + + py.run("cache[ptr] = obj; cache[id(obj)] = ptr", None, Some(locals)) + } + + // py to native methods + + fn from_py_to_native_cache<'p>(&self, py: Python<'p>, obj: &PyAny) -> PyResult { + let locals = [("cache", self.cache.clone()), ("key", obj.to_object(py))].into_py_dict(py); + py.eval("cache.get(id(key))", None, Some(locals))?.extract() + } + + /// copy this python object into this `Arena` if it's not yet in the cache + /// (otherwise it returns the previously cached object) + pub fn populate_native( + &self, + py: Python, + obj: &PyAny, + allocator: &mut Allocator, + ) -> PyResult { + // items in `pending` are already in the stack of things to be converted + // if they appear again, we have an illegal cycle and must fail + + let mut pending: HashSet = HashSet::new(); + + apply_to_tree(obj, move |obj| { + // is it in cache yet? + if self.from_py_to_native_cache(py, obj).is_ok() { + // yep, we're done + return Ok(None); + } + + // it's not in the cache + + match sexp_for_obj(obj)? { + PySExp::Atom(atom) => { + let blob: &[u8] = atom.extract()?; + let ptr = allocator.new_atom(blob).unwrap(); + self.add(py, obj, ptr)?; + + Ok(None) + } + PySExp::Pair(p0, p1) => { + let ptr_0: PyResult = self.from_py_to_native_cache(py, p0); + let ptr_1: PyResult = self.from_py_to_native_cache(py, p1); + + let as_obj = id_for_pyany(py, obj)?; + + if let (Ok(ptr_0), Ok(ptr_1)) = (ptr_0, ptr_1) { + let ptr = allocator.new_pair(ptr_0, ptr_1).unwrap(); + self.add(py, obj, ptr)?; + + pending.remove(&as_obj); + Ok(None) + } else { + if pending.contains(&as_obj) { + let locals = Some([("obj", obj)].into_py_dict(py)); + py.run( + "raise ValueError(f'illegal clvm object loop {obj}')", + None, + locals, + )?; + panic!(); + } + pending.insert(as_obj); + + Ok(Some((p0, p1))) + } + } + } + })?; + + self.from_py_to_native_cache(py, obj) + } + + pub fn native_for_py( + &self, + py: Python, + obj: &PyAny, + allocator: &mut Allocator, + ) -> PyResult { + self.from_py_to_native_cache(py, obj) + .or_else(|_err| self.populate_native(py, obj, allocator)) + } + + // native to py methods + + fn from_native_to_py_cache<'p>(&self, py: Python<'p>, ptr: NodePtr) -> PyResult<&'p PyAny> { + let locals = [("cache", self.cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); + py.eval("cache[key]", None, Some(locals))?.extract() + } + + fn populate_python<'p>( + &self, + py: Python<'p>, + ptr: NodePtr, + allocator: &mut Allocator, + ) -> PyResult<&'p PyAny> { + apply_to_tree(ptr, move |ptr| { + // is it in cache yet? + if self.from_native_to_py_cache(py, ptr).is_ok() { + // yep, we're done + return Ok(None); + } + + // it's not in the cache + + match allocator.sexp(ptr) { + SExp::Atom(a) => { + // it's an atom, so we just populate cache directly + let blob = allocator.buf(&a); + let py_bytes = PyBytes::new(py, blob); + self.add(py, self.to_python.as_ref(py).call1((py_bytes,))?, ptr)?; + Ok(None) + } + SExp::Pair(ptr_1, ptr_2) => { + // we can only create this if the children are in the cache + // Let's find out + let locals = [ + ("cache", self.cache.clone()), + ("p1", ptr_1.to_object(py)), + ("p2", ptr_2.to_object(py)), + ] + .into_py_dict(py); + + let pair: PyResult<&PyAny> = + py.eval("(cache[p1], cache[p2])", None, Some(locals)); + + match pair { + // the children aren't in the cache, keep drilling down + Err(_) => Ok(Some((ptr_1, ptr_2))), + + // the children are in the cache, create new node & populate cache with it + Ok(tuple) => { + let (_p1, _p2): (&PyAny, &PyAny) = tuple.extract()?; + self.add(py, self.to_python.as_ref(py).call1((tuple,))?, ptr)?; + Ok(None) + } + } + } + } + })?; + + self.from_native_to_py_cache(py, ptr) + } + + pub fn py_for_native<'p>( + &self, + py: Python<'p>, + ptr: NodePtr, + allocator: &mut Allocator, + ) -> PyResult<&'p PyAny> { + self.from_native_to_py_cache(py, ptr) + .or_else(|_err| self.populate_python(py, ptr, allocator)) + } +} + +fn id_for_pyany(py: Python, obj: &PyAny) -> PyResult { + let locals = Some([("obj", obj)].into_py_dict(py)); + py.eval("id(obj)", None, locals)?.extract() +} + +fn apply_to_tree(node: T, mut apply: F) -> PyResult<()> +where + F: FnMut(T) -> PyResult>, + T: Clone, +{ + let mut items = vec![node]; + loop { + let t = items.pop(); + if let Some(obj) = t { + if let Some((p0, p1)) = apply(obj.clone())? { + items.push(obj); + items.push(p0); + items.push(p1); + } + } else { + break; + } + } + Ok(()) +} diff --git a/src/py/dialect.rs b/src/py/dialect.rs index d3afeaec..fde7f6c0 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -20,6 +20,8 @@ use super::f_table::FLookup; use super::f_table::OpFn; use super::native_op::NativeOp; +type MultiOpFn = fn(&mut Allocator, NodePtr, NodePtr, Cost) -> Response; + #[pyclass] #[derive(Clone)] pub struct PyMultiOpFn { @@ -32,8 +34,6 @@ impl PyMultiOpFn { } } -pub type MultiOpFn = fn(&mut Allocator, NodePtr, NodePtr, Cost) -> Response; - #[derive(Clone)] pub enum MultiOpFnE { Python(PyObject), @@ -168,8 +168,8 @@ fn pre_eval_callback( args: NodePtr, ) -> PyResult { // call the python `pre_eval` object and return the python object yielded - let program_obj = arena.py_for_native(py, program, allocator)?; - let args_obj = arena.py_for_native(py, args, allocator)?; + let program_obj = arena.cache.py_for_native(py, program, allocator)?; + let args_obj = arena.cache.py_for_native(py, args, allocator)?; let post_eval_obj = pre_eval_obj .call1(py, (program_obj, args_obj))? .to_object(py); @@ -214,7 +214,8 @@ impl Dialect { let local_arena = &arena; let post_eval: Box = Box::new(move |allocator: &mut Allocator, result_ptr: i32| { - if let Ok(r) = local_arena.py_for_native(py, result_ptr, allocator) + if let Ok(r) = + local_arena.cache.py_for_native(py, result_ptr, allocator) { // invoke the python `PostEval` callback let _r = post_eval_obj.call1(py, (r.to_object(py),)); @@ -242,11 +243,12 @@ impl Dialect { match r { Ok(reduction) => { - let r = arena.py_for_native(py, reduction.1, allocator)?; + let r = arena.cache.py_for_native(py, reduction.1, allocator)?; Ok((reduction.0, r)) } Err(eval_err) => { let node: PyObject = arena + .cache .py_for_native(py, eval_err.0, allocator)? .to_object(py); let s: String = eval_err.1; @@ -276,7 +278,7 @@ impl DialectRunningContext<'_> { ) -> Response { Python::with_gil(|py| { let r = unwrap_or_eval_err( - Arena::py_for_native(&self.arena, py, args, allocator), + self.arena.cache.py_for_native(py, args, allocator), args, "can't uncache", )?; @@ -297,7 +299,7 @@ impl DialectRunningContext<'_> { let clvm_object: &PyAny = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = Arena::native_for_py(&self.arena, py, clvm_object, allocator); + let r = self.arena.cache.native_for_py(py, clvm_object, allocator); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } diff --git a/src/py/error_bridge.rs b/src/py/error_bridge.rs index 337c5b3b..164cf464 100644 --- a/src/py/error_bridge.rs +++ b/src/py/error_bridge.rs @@ -16,7 +16,7 @@ pub fn eval_err_for_pyerr<'p>( let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; let sexp: &PyAny = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = Arena::native_for_py(arena, py, sexp, allocator)?; + let node: i32 = arena.cache.native_for_py(py, sexp, allocator)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) } diff --git a/src/py/mod.rs b/src/py/mod.rs index bdf3c7d7..fd9f3d8b 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -1,5 +1,6 @@ pub mod api; pub mod arena; +pub mod bridge_cache; pub mod dialect; pub mod error_bridge; pub mod f_table; diff --git a/src/py/native_op.rs b/src/py/native_op.rs index eb350ec0..b5a84512 100644 --- a/src/py/native_op.rs +++ b/src/py/native_op.rs @@ -38,11 +38,11 @@ impl NativeOp { let r = (self.op)(allocator, ptr, _max_cost); match r { Ok(Reduction(cost, ptr)) => { - let r = arena.py_for_native(py, ptr, allocator)?; + let r = arena.cache.py_for_native(py, ptr, allocator)?; Ok((cost, r.to_object(py))) } Err(_err) => { - let r = arena.py_for_native(py, ptr, allocator)?; + let r = arena.cache.py_for_native(py, ptr, allocator)?; match raise_eval_error( py, PyString::new(py, "problem in suboperator"), diff --git a/src/py/run_program.rs b/src/py/run_program.rs index 9713186b..ef32fc8b 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -78,6 +78,7 @@ pub fn deserialize_and_run_program( Ok(reduction) => Ok(( reduction.0, arena_borrowed + .cache .py_for_native(py, reduction.1, allocator)? .to_object(py), )), From ee8ebaba02c21671977164b3d3fad9e4f88ed9a5 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 1 Jul 2021 15:41:12 -0700 Subject: [PATCH 75/76] Improve `BridgeCache`. --- src/py/api.rs | 2 +- src/py/arena.rs | 36 ++++++++++++++++++++++++--------- src/py/bridge_cache.rs | 46 ++++++++++++++++-------------------------- src/py/dialect.rs | 19 +++++++---------- src/py/error_bridge.rs | 2 +- src/py/native_op.rs | 4 ++-- src/py/run_program.rs | 3 +-- 7 files changed, 55 insertions(+), 57 deletions(-) diff --git a/src/py/api.rs b/src/py/api.rs index 33f5cb0f..ea5bf29b 100644 --- a/src/py/api.rs +++ b/src/py/api.rs @@ -70,7 +70,7 @@ fn serialize_to_bytes<'p>(py: Python<'p>, sexp: &PyAny) -> PyResult<&'p PyBytes> let mut allocator_refcell: RefMut = arena.allocator(); let allocator: &mut Allocator = &mut allocator_refcell as &mut Allocator; - let ptr = arena.cache.native_for_py(py, sexp, allocator)?; + let ptr = arena.as_native(py, allocator, sexp)?; let node = Node::new(allocator, ptr); let s: Vec = node_to_bytes(&node)?; diff --git a/src/py/arena.rs b/src/py/arena.rs index 286f0d51..ebcecd40 100644 --- a/src/py/arena.rs +++ b/src/py/arena.rs @@ -8,14 +8,13 @@ use pyo3::prelude::pyclass; use pyo3::prelude::*; use super::bridge_cache::BridgeCache; -use crate::allocator::Allocator; +use crate::allocator::{Allocator, NodePtr}; use crate::serialize::node_from_bytes; #[pyclass(subclass, unsendable)] pub struct Arena { arena: RefCell, - pub cache: BridgeCache, - // TODO: make this private, remove this `pub` + cache: BridgeCache, } #[pymethods] @@ -32,22 +31,21 @@ impl Arena { pub fn deserialize<'p>(&self, py: Python<'p>, blob: &[u8]) -> PyResult<&'p PyAny> { let allocator: &mut Allocator = &mut self.allocator() as &mut Allocator; let ptr = node_from_bytes(allocator, blob)?; - self.cache.py_for_native(py, ptr, allocator) + self.as_python(py, allocator, ptr) } /// copy this python object into this `Arena` if it's not yet in the cache /// (otherwise it returns the previously cached object) pub fn include<'p>(&self, py: Python<'p>, obj: &'p PyAny) -> PyResult<&'p PyAny> { - let ptr = Self::ptr_for_obj(self, py, obj)?; - self.cache - .py_for_native(py, ptr, &mut self.allocator() as &mut Allocator) + let allocator = &mut self.allocator(); + let ptr = self.as_native(py, allocator, obj)?; + self.as_python(py, allocator, ptr) } /// copy this python object into this `Arena` if it's not yet in the cache /// (otherwise it returns the previously cached object) pub fn ptr_for_obj(&self, py: Python, obj: &PyAny) -> PyResult { - let allocator: &mut Allocator = &mut self.allocator() as &mut Allocator; - self.cache.populate_native(py, obj, allocator) + self.as_native(py, &mut self.allocator(), obj) } } @@ -61,10 +59,28 @@ impl Arena { } pub fn obj_for_ptr<'p>(&self, py: Python<'p>, ptr: i32) -> PyResult<&'p PyAny> { - self.cache.py_for_native(py, ptr, &mut self.allocator()) + self.as_python(py, &mut self.allocator(), ptr) } pub fn allocator(&self) -> RefMut { self.arena.borrow_mut() } + + pub fn as_native( + &self, + py: Python, + allocator: &mut Allocator, + obj: &PyAny, + ) -> PyResult { + self.cache.as_native(py, allocator, obj) + } + + pub fn as_python<'p>( + &self, + py: Python<'p>, + allocator: &mut Allocator, + ptr: NodePtr, + ) -> PyResult<&'p PyAny> { + self.cache.as_python(py, allocator, ptr) + } } diff --git a/src/py/bridge_cache.rs b/src/py/bridge_cache.rs index 88b421e0..d4fa4c8b 100644 --- a/src/py/bridge_cache.rs +++ b/src/py/bridge_cache.rs @@ -44,21 +44,9 @@ impl BridgeCache { }) } - /// copy this python object into this `BridgeCache` if it's not yet in the cache - /// (otherwise it returns the previously cached object) - pub fn include<'p>( - &self, - py: Python<'p>, - allocator: &mut Allocator, - obj: &'p PyAny, - ) -> PyResult<&'p PyAny> { - let ptr = self.populate_native(py, obj, allocator)?; - self.py_for_native(py, ptr, allocator) - } - /// add a python object <-> native object mapping /// to the cache, in both directions - pub fn add(&self, py: Python, obj: &PyAny, ptr: NodePtr) -> PyResult<()> { + fn add(&self, py: Python, obj: &PyAny, ptr: NodePtr) -> PyResult<()> { let locals = [ ("cache", self.cache.clone()), ("obj", obj.to_object(py)), @@ -71,14 +59,14 @@ impl BridgeCache { // py to native methods - fn from_py_to_native_cache<'p>(&self, py: Python<'p>, obj: &PyAny) -> PyResult { + fn native_from_cache<'p>(&self, py: Python<'p>, obj: &PyAny) -> PyResult { let locals = [("cache", self.cache.clone()), ("key", obj.to_object(py))].into_py_dict(py); - py.eval("cache.get(id(key))", None, Some(locals))?.extract() + py.eval("cache[id(key)]", None, Some(locals))?.extract() } /// copy this python object into this `Arena` if it's not yet in the cache /// (otherwise it returns the previously cached object) - pub fn populate_native( + fn populate_native( &self, py: Python, obj: &PyAny, @@ -91,7 +79,7 @@ impl BridgeCache { apply_to_tree(obj, move |obj| { // is it in cache yet? - if self.from_py_to_native_cache(py, obj).is_ok() { + if self.native_from_cache(py, obj).is_ok() { // yep, we're done return Ok(None); } @@ -107,8 +95,8 @@ impl BridgeCache { Ok(None) } PySExp::Pair(p0, p1) => { - let ptr_0: PyResult = self.from_py_to_native_cache(py, p0); - let ptr_1: PyResult = self.from_py_to_native_cache(py, p1); + let ptr_0: PyResult = self.native_from_cache(py, p0); + let ptr_1: PyResult = self.native_from_cache(py, p1); let as_obj = id_for_pyany(py, obj)?; @@ -136,22 +124,22 @@ impl BridgeCache { } })?; - self.from_py_to_native_cache(py, obj) + self.native_from_cache(py, obj) } - pub fn native_for_py( + pub fn as_native( &self, py: Python, - obj: &PyAny, allocator: &mut Allocator, + obj: &PyAny, ) -> PyResult { - self.from_py_to_native_cache(py, obj) + self.native_from_cache(py, obj) .or_else(|_err| self.populate_native(py, obj, allocator)) } // native to py methods - fn from_native_to_py_cache<'p>(&self, py: Python<'p>, ptr: NodePtr) -> PyResult<&'p PyAny> { + fn python_from_cache<'p>(&self, py: Python<'p>, ptr: NodePtr) -> PyResult<&'p PyAny> { let locals = [("cache", self.cache.clone()), ("key", ptr.to_object(py))].into_py_dict(py); py.eval("cache[key]", None, Some(locals))?.extract() } @@ -164,7 +152,7 @@ impl BridgeCache { ) -> PyResult<&'p PyAny> { apply_to_tree(ptr, move |ptr| { // is it in cache yet? - if self.from_native_to_py_cache(py, ptr).is_ok() { + if self.python_from_cache(py, ptr).is_ok() { // yep, we're done return Ok(None); } @@ -207,16 +195,16 @@ impl BridgeCache { } })?; - self.from_native_to_py_cache(py, ptr) + self.python_from_cache(py, ptr) } - pub fn py_for_native<'p>( + pub fn as_python<'p>( &self, py: Python<'p>, - ptr: NodePtr, allocator: &mut Allocator, + ptr: NodePtr, ) -> PyResult<&'p PyAny> { - self.from_native_to_py_cache(py, ptr) + self.python_from_cache(py, ptr) .or_else(|_err| self.populate_python(py, ptr, allocator)) } } diff --git a/src/py/dialect.rs b/src/py/dialect.rs index fde7f6c0..5da4e8de 100644 --- a/src/py/dialect.rs +++ b/src/py/dialect.rs @@ -168,8 +168,8 @@ fn pre_eval_callback( args: NodePtr, ) -> PyResult { // call the python `pre_eval` object and return the python object yielded - let program_obj = arena.cache.py_for_native(py, program, allocator)?; - let args_obj = arena.cache.py_for_native(py, args, allocator)?; + let program_obj = arena.as_python(py, allocator, program)?; + let args_obj = arena.as_python(py, allocator, args)?; let post_eval_obj = pre_eval_obj .call1(py, (program_obj, args_obj))? .to_object(py); @@ -214,9 +214,7 @@ impl Dialect { let local_arena = &arena; let post_eval: Box = Box::new(move |allocator: &mut Allocator, result_ptr: i32| { - if let Ok(r) = - local_arena.cache.py_for_native(py, result_ptr, allocator) - { + if let Ok(r) = local_arena.as_python(py, allocator, result_ptr) { // invoke the python `PostEval` callback let _r = post_eval_obj.call1(py, (r.to_object(py),)); } @@ -243,14 +241,11 @@ impl Dialect { match r { Ok(reduction) => { - let r = arena.cache.py_for_native(py, reduction.1, allocator)?; + let r = arena.as_python(py, allocator, reduction.1)?; Ok((reduction.0, r)) } Err(eval_err) => { - let node: PyObject = arena - .cache - .py_for_native(py, eval_err.0, allocator)? - .to_object(py); + let node: PyObject = arena.as_python(py, allocator, eval_err.0)?.to_object(py); let s: String = eval_err.1; let s1: &str = &s; let msg: &PyString = PyString::new(py, s1); @@ -278,7 +273,7 @@ impl DialectRunningContext<'_> { ) -> Response { Python::with_gil(|py| { let r = unwrap_or_eval_err( - self.arena.cache.py_for_native(py, args, allocator), + self.arena.as_python(py, allocator, args), args, "can't uncache", )?; @@ -299,7 +294,7 @@ impl DialectRunningContext<'_> { let clvm_object: &PyAny = unwrap_or_eval_err(pair.get_item(1).extract(), args, "expected node")?; - let r = self.arena.cache.native_for_py(py, clvm_object, allocator); + let r = self.arena.as_native(py, allocator, clvm_object); let node: i32 = unwrap_or_eval_err(r, args, "can't find in int allocator")?; Ok(Reduction(i0 as Cost, node)) } diff --git a/src/py/error_bridge.rs b/src/py/error_bridge.rs index 164cf464..d1e16dae 100644 --- a/src/py/error_bridge.rs +++ b/src/py/error_bridge.rs @@ -16,7 +16,7 @@ pub fn eval_err_for_pyerr<'p>( let args: &PyTuple = pyerr.pvalue(py).getattr("args")?.extract()?; let arg0: &PyString = args.get_item(0).extract()?; let sexp: &PyAny = pyerr.pvalue(py).getattr("_sexp")?.extract()?; - let node: i32 = arena.cache.native_for_py(py, sexp, allocator)?; + let node: i32 = arena.as_native(py, allocator, sexp)?; let s: String = arg0.to_str()?.to_string(); Ok(EvalErr(node, s)) } diff --git a/src/py/native_op.rs b/src/py/native_op.rs index b5a84512..5093c01e 100644 --- a/src/py/native_op.rs +++ b/src/py/native_op.rs @@ -38,11 +38,11 @@ impl NativeOp { let r = (self.op)(allocator, ptr, _max_cost); match r { Ok(Reduction(cost, ptr)) => { - let r = arena.cache.py_for_native(py, ptr, allocator)?; + let r = arena.as_python(py, allocator, ptr)?; Ok((cost, r.to_object(py))) } Err(_err) => { - let r = arena.cache.py_for_native(py, ptr, allocator)?; + let r = arena.as_python(py, allocator, ptr)?; match raise_eval_error( py, PyString::new(py, "problem in suboperator"), diff --git a/src/py/run_program.rs b/src/py/run_program.rs index ef32fc8b..3df71c06 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -78,8 +78,7 @@ pub fn deserialize_and_run_program( Ok(reduction) => Ok(( reduction.0, arena_borrowed - .cache - .py_for_native(py, reduction.1, allocator)? + .as_python(py, allocator, reduction.1)? .to_object(py), )), Err(eval_err) => { From 50b3536784e81766aeb9cdf01374e04b1f25a4db Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 2 Jul 2021 14:37:22 -0700 Subject: [PATCH 76/76] Use `&[u8]` for opcode instead of `u8`. --- src/py/run_program.rs | 16 ++++++++-------- src/run_program.rs | 38 ++++++++++++++++---------------------- src/tests.rs | 2 -- 3 files changed, 24 insertions(+), 32 deletions(-) diff --git a/src/py/run_program.rs b/src/py/run_program.rs index d5e3c851..4652dffd 100644 --- a/src/py/run_program.rs +++ b/src/py/run_program.rs @@ -120,7 +120,7 @@ pub fn deserialize_and_run_program( let mut allocator = Allocator::new(); let f_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); let strict: bool = (flags & STRICT_MODE) != 0; - let f: Box = Box::new(OperatorHandlerWithMode { f_lookup, strict }); + let f = OperatorHandlerWithMode { f_lookup, strict }; let program = node_from_bytes(&mut allocator, program)?; let args = node_from_bytes(&mut allocator, args)?; @@ -129,10 +129,10 @@ pub fn deserialize_and_run_program( &mut allocator, program, args, - quote_kw, - apply_kw, + &[quote_kw], + &[apply_kw], max_cost, - f, + &f, None, ) }); @@ -187,7 +187,7 @@ pub fn deserialize_and_run_program2( let mut allocator = Allocator::new(); let f_lookup = f_lookup_for_hashmap(opcode_lookup_by_name); let strict: bool = (flags & STRICT_MODE) != 0; - let f: Box = Box::new(OperatorHandlerWithMode { f_lookup, strict }); + let f = OperatorHandlerWithMode { f_lookup, strict }; let program = node_from_bytes(&mut allocator, program)?; let args = node_from_bytes(&mut allocator, args)?; @@ -196,10 +196,10 @@ pub fn deserialize_and_run_program2( &mut allocator, program, args, - quote_kw, - apply_kw, + &[quote_kw], + &[apply_kw], max_cost, - f, + &f, None, ) }); diff --git a/src/run_program.rs b/src/run_program.rs index f1ffb1f2..70d9c54f 100644 --- a/src/run_program.rs +++ b/src/run_program.rs @@ -41,9 +41,9 @@ enum Operation { pub struct RunProgramContext<'a> { allocator: &'a mut Allocator, - quote_kw: u8, - apply_kw: u8, - operator_lookup: Box, + quote_kw: &'a [u8], + apply_kw: &'a [u8], + operator_lookup: &'a dyn OperatorHandler, pre_eval: Option, posteval_stack: Vec>, val_stack: Vec, @@ -113,7 +113,7 @@ fn traverse_path(allocator: &Allocator, node_index: &[u8], args: NodePtr) -> Res return Err(EvalErr(arg_list, "path into atom".into())); } SExp::Pair(left, right) => { - arg_list = *(if is_bit_set { &right } else { &left }); + arg_list = if is_bit_set { right } else { left }; } } if bitmask == 0x80 { @@ -142,9 +142,9 @@ fn augment_cost_errors(r: Result, max_cost: NodePtr) -> Result RunProgramContext<'a> { fn new( allocator: &'a mut Allocator, - quote_kw: u8, - apply_kw: u8, - operator_lookup: Box, + quote_kw: &'a [u8], + apply_kw: &'a [u8], + operator_lookup: &'a dyn OperatorHandler, pre_eval: Option, ) -> Self { RunProgramContext { @@ -178,10 +178,7 @@ impl<'a> RunProgramContext<'a> { } } -impl<'a> RunProgramContext<'a> -where - NodePtr: 'static, -{ +impl<'a> RunProgramContext<'a> { fn eval_op_atom( &mut self, op_buf: &AtomBuf, @@ -191,7 +188,7 @@ where ) -> Result { let op_atom = self.allocator.buf(op_buf); // special case check for quote - if op_atom.len() == 1 && op_atom[0] == self.quote_kw { + if op_atom.len() == 1 && op_atom == self.quote_kw { self.push(operand_list); Ok(QUOTE_COST) } else { @@ -282,7 +279,7 @@ where return err(operator, "internal error"); } let op_atom = self.allocator.atom(operator); - if op_atom.len() == 1 && op_atom[0] == self.apply_kw { + if op_atom.len() == 1 && op_atom == self.apply_kw { let operand_list = Node::new(self.allocator, operand_list); if operand_list.arg_count_is(2) { let new_operator = operand_list.first()?; @@ -347,19 +344,16 @@ where } #[allow(clippy::too_many_arguments)] -pub fn run_program( - allocator: &mut Allocator, +pub fn run_program<'a>( + allocator: &'a mut Allocator, program: NodePtr, args: NodePtr, - quote_kw: u8, - apply_kw: u8, + quote_kw: &'a [u8], + apply_kw: &'a [u8], max_cost: Cost, - operator_lookup: Box, + operator_lookup: &'a dyn OperatorHandler, pre_eval: Option, -) -> Response -where - NodePtr: 'static, -{ +) -> Response { let mut rpc = RunProgramContext::new(allocator, quote_kw, apply_kw, operator_lookup, pre_eval); rpc.run_program(program, args, max_cost) } diff --git a/src/tests.rs b/src/tests.rs index 4e3e4666..25754473 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -172,9 +172,7 @@ fn node_to_hex(node: &Node) -> String { fn do_test_run_program(input_as_hex: &str, expected_as_hex: &str) -> () { let mut a = Allocator::new(); let n = node_from_hex(&a, input_as_hex); - println!("n = {:?}", n); let r = do_run_program(&n, &null); - println!("r = {:?}", r); assert_eq!(node_to_hex(&r), expected_as_hex); }