feat: update pyo3 (#308)

* feat: update pyo3

* update workflows

* python version

* update minimum

* update pyproject

* fix pipeline

bindings/python/Cargo.toml

@@ -12,9 +12,9 @@ crate-type = ["cdylib"]
 [dependencies]
 anyhow = { workspace = true }
 either = "1.13"
-pyo3 = { version = "0.20", features = ["anyhow", "serde"] }
-pyo3-asyncio = { version = "0.20", features = ["tokio-runtime"] }
-pythonize = "0.20"
+pyo3 = { version = "0.23", features = ["anyhow", "serde"] }
+pyo3-async-runtimes = { version = "0.23", features = ["tokio-runtime"] }
+pythonize = "0.23"
 json_dotpath = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }

bindings/python/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["maturin>=0.14,<0.15"]
+requires = ["maturin>=1,<2"]
 build-backend = "maturin"
 
 [project]

bindings/python/src/custom_node.rs

@@ -1,8 +1,8 @@
 use anyhow::anyhow;
 use either::Either;
 use pyo3::types::PyDict;
-use pyo3::{PyObject, PyResult, Python};
-use pyo3_asyncio::tokio;
+use pyo3::{Bound, IntoPyObjectExt, Py, PyAny, PyObject, PyResult, Python};
+use pyo3_async_runtimes::tokio;
 use pythonize::depythonize;
 
 use zen_engine::handler::custom_node_adapter::{CustomNodeAdapter, CustomNodeRequest};
@@ -11,10 +11,10 @@ use zen_engine::handler::node::{NodeResponse, NodeResult};
 use crate::types::PyNodeRequest;
 
 #[derive(Default)]
-pub(crate) struct PyCustomNode(Option<PyObject>);
+pub(crate) struct PyCustomNode(Option<Py<PyAny>>);
 
-impl From<PyObject> for PyCustomNode {
-    fn from(value: PyObject) -> Self {
+impl From<Py<PyAny>> for PyCustomNode {
+    fn from(value: Py<PyAny>) -> Self {
         Self(Some(value))
     }
 }
@@ -26,8 +26,8 @@ impl From<Option<PyObject>> for PyCustomNode {
 }
 
 fn extract_custom_node_response(py: Python<'_>, result: PyObject) -> NodeResult {
-    let dict = result.extract::<&PyDict>(py)?;
-    let response: NodeResponse = depythonize(dict)?;
+    let dict = result.extract::<Bound<'_, PyDict>>(py)?;
+    let response: NodeResponse = depythonize(&dict)?;
     Ok(response)
 }
 
@@ -45,7 +45,7 @@ impl CustomNodeAdapter for PyCustomNode {
                 return Ok(Either::Left(extract_custom_node_response(py, result)));
             }
 
-            let result_future = tokio::into_future(result.as_ref(py))?;
+            let result_future = tokio::into_future(result.into_bound_py_any(py)?)?;
             return Ok(Either::Right(result_future));
         });
 

bindings/python/src/decision.rs

@@ -2,8 +2,8 @@ use std::sync::Arc;
 
 use anyhow::{anyhow, Context};
 use pyo3::types::PyDict;
-use pyo3::{pyclass, pymethods, PyAny, PyObject, PyResult, Python, ToPyObject};
-use pyo3_asyncio::tokio;
+use pyo3::{pyclass, pymethods, Bound, IntoPyObjectExt, Py, PyAny, PyResult, Python};
+use pyo3_async_runtimes::tokio;
 use pythonize::depythonize;
 use serde_json::Value;
 use zen_engine::{Decision, EvaluationOptions};
@@ -25,7 +25,13 @@ impl From<Decision<PyDecisionLoader, PyCustomNode>> for PyZenDecision {
 
 #[pymethods]
 impl PyZenDecision {
-    pub fn evaluate(&self, py: Python, ctx: &PyDict, opts: Option<&PyDict>) -> PyResult<PyObject> {
+    #[pyo3(signature = (ctx, opts=None))]
+    pub fn evaluate(
+        &self,
+        py: Python,
+        ctx: &Bound<'_, PyDict>,
+        opts: Option<&Bound<'_, PyDict>>,
+    ) -> PyResult<Py<PyAny>> {
         let context: Value = depythonize(ctx).context("Failed to convert dict")?;
         let options: PyZenEvaluateOptions = if let Some(op) = opts {
             depythonize(op).context("Failed to convert dict")?
@@ -46,15 +52,16 @@ impl PyZenDecision {
         })?;
 
         let value = serde_json::to_value(&result).context("Fail")?;
-        Ok(PyValue(value).to_object(py))
+        PyValue(value).into_py_any(py)
     }
 
+    #[pyo3(signature = (ctx, opts=None))]
     pub fn async_evaluate<'py>(
         &'py self,
         py: Python<'py>,
-        ctx: &PyDict,
-        opts: Option<&PyDict>,
-    ) -> PyResult<&PyAny> {
+        ctx: &Bound<'_, PyDict>,
+        opts: Option<&Bound<'_, PyDict>>,
+    ) -> PyResult<Py<PyAny>> {
         let context: Value = depythonize(ctx).context("Failed to convert dict")?;
         let options: PyZenEvaluateOptions = if let Some(op) = opts {
             depythonize(op).context("Failed to convert dict")?
@@ -63,7 +70,7 @@ impl PyZenDecision {
         };
 
         let decision = self.0.clone();
-        tokio::future_into_py(py, async move {
+        let result = tokio::future_into_py(py, async move {
             let result = futures::executor::block_on(decision.evaluate_with_opts(
                 context.into(),
                 EvaluationOptions {
@@ -77,8 +84,10 @@ impl PyZenDecision {
 
             let value = serde_json::to_value(result).context("Failed to serialize result")?;
 
-            Python::with_gil(|py| Ok(PyValue(value).to_object(py)))
-        })
+            Python::with_gil(|py| PyValue(value).into_py_any(py))
+        })?;
+
+        Ok(result.unbind())
     }
 
     pub fn validate(&self) -> PyResult<()> {

bindings/python/src/engine.rs

@@ -1,9 +1,10 @@
 use std::sync::Arc;
 
 use anyhow::{anyhow, Context};
+use pyo3::prelude::PyDictMethods;
 use pyo3::types::PyDict;
-use pyo3::{pyclass, pymethods, PyAny, PyObject, PyResult, Python, ToPyObject};
-use pyo3_asyncio::tokio;
+use pyo3::{pyclass, pymethods, Bound, IntoPyObjectExt, Py, PyAny, PyResult, Python};
+use pyo3_async_runtimes::tokio;
 use pythonize::depythonize;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
@@ -51,18 +52,19 @@ impl Default for PyZenEngine {
 #[pymethods]
 impl PyZenEngine {
     #[new]
-    pub fn new(maybe_options: Option<&PyDict>) -> PyResult<Self> {
+    #[pyo3(signature = (maybe_options=None))]
+    pub fn new(maybe_options: Option<&Bound<'_, PyDict>>) -> PyResult<Self> {
         let Some(options) = maybe_options else {
             return Ok(Default::default());
         };
 
         let loader = match options.get_item("loader")? {
-            Some(loader) => Some(Python::with_gil(|py| loader.to_object(py))),
+            Some(loader) => Some(Python::with_gil(|py| loader.into_py_any(py))?),
             None => None,
         };
 
         let custom_node = match options.get_item("customHandler")? {
-            Some(custom_node) => Some(Python::with_gil(|py| custom_node.to_object(py))),
+            Some(custom_node) => Some(Python::with_gil(|py| custom_node.into_py_any(py))?),
             None => None,
         };
 
@@ -75,13 +77,14 @@ impl PyZenEngine {
         })
     }
 
+    #[pyo3(signature = (key, ctx, opts=None))]
     pub fn evaluate(
         &self,
         py: Python,
         key: String,
-        ctx: &PyDict,
-        opts: Option<&PyDict>,
-    ) -> PyResult<PyObject> {
+        ctx: &Bound<'_, PyDict>,
+        opts: Option<&Bound<'_, PyDict>>,
+    ) -> PyResult<Py<PyAny>> {
         let context: Value = depythonize(ctx).context("Failed to convert dict")?;
         let options: PyZenEvaluateOptions = if let Some(op) = opts {
             depythonize(op).context("Failed to convert dict")?
@@ -103,16 +106,17 @@ impl PyZenEngine {
         })?;
 
         let value = serde_json::to_value(&result).context("Failed to serialize result")?;
-        Ok(PyValue(value).to_object(py))
+        PyValue(value).into_py_any(py)
     }
 
+    #[pyo3(signature = (key, ctx, opts=None))]
     pub fn async_evaluate<'py>(
         &'py self,
         py: Python<'py>,
         key: String,
-        ctx: &PyDict,
-        opts: Option<&PyDict>,
-    ) -> PyResult<&PyAny> {
+        ctx: &Bound<'_, PyDict>,
+        opts: Option<&Bound<'_, PyDict>>,
+    ) -> PyResult<Py<PyAny>> {
         let context: Value = depythonize(ctx).context("Failed to convert dict")?;
         let options: PyZenEvaluateOptions = if let Some(op) = opts {
             depythonize(op).context("Failed to convert dict")?
@@ -121,7 +125,7 @@ impl PyZenEngine {
         };
 
         let graph = self.graph.clone();
-        tokio::future_into_py(py, async move {
+        let result = tokio::future_into_py(py, async move {
             let result = futures::executor::block_on(graph.evaluate_with_opts(
                 key,
                 context.into(),
@@ -136,8 +140,10 @@ impl PyZenEngine {
 
             let value = serde_json::to_value(result).context("Failed to serialize result")?;
 
-            Python::with_gil(|py| Ok(PyValue(value).to_object(py)))
-        })
+            Python::with_gil(|py| PyValue(value).into_py_any(py))
+        })?;
+
+        Ok(result.unbind())
     }
 
     pub fn create_decision(&self, content: String) -> PyResult<PyZenDecision> {

bindings/python/src/expression.rs

@@ -1,17 +1,18 @@
 use anyhow::{anyhow, Context};
 use pyo3::types::PyDict;
-use pyo3::{pyfunction, PyObject, PyResult, Python, ToPyObject};
+use pyo3::{pyfunction, Bound, IntoPyObjectExt, Py, PyAny, PyResult, Python};
 use pythonize::depythonize;
 use serde_json::Value;
 
 use crate::value::PyValue;
 
 #[pyfunction]
+#[pyo3(signature = (expression, ctx=None))]
 pub fn evaluate_expression(
     py: Python,
     expression: String,
-    ctx: Option<&PyDict>,
-) -> PyResult<PyObject> {
+    ctx: Option<&Bound<'_, PyDict>>,
+) -> PyResult<Py<PyAny>> {
     let context = ctx
         .map(|ctx| depythonize(ctx))
         .transpose()
@@ -21,11 +22,11 @@ pub fn evaluate_expression(
     let result = zen_expression::evaluate_expression(expression.as_str(), context.into())
         .map_err(|e| anyhow!(serde_json::to_string(&e).unwrap_or_else(|_| e.to_string())))?;
 
-    Ok(PyValue(result.to_value()).to_object(py))
+    PyValue(result.to_value()).into_py_any(py)
 }
 
 #[pyfunction]
-pub fn evaluate_unary_expression(expression: String, ctx: &PyDict) -> PyResult<bool> {
+pub fn evaluate_unary_expression(expression: String, ctx: &Bound<'_, PyDict>) -> PyResult<bool> {
     let context: Value = depythonize(ctx).context("Failed to convert context")?;
 
     let result = zen_expression::evaluate_unary_expression(expression.as_str(), context.into())
@@ -35,11 +36,15 @@ pub fn evaluate_unary_expression(expression: String, ctx: &PyDict) -> PyResult<b
 }
 
 #[pyfunction]
-pub fn render_template(py: Python, template: String, ctx: &PyDict) -> PyResult<PyObject> {
+pub fn render_template(
+    py: Python,
+    template: String,
+    ctx: &Bound<'_, PyDict>,
+) -> PyResult<Py<PyAny>> {
     let context: Value = depythonize(ctx).context("Failed to convert context")?;
 
     let result = zen_tmpl::render(template.as_str(), context.into())
         .map_err(|e| anyhow!(serde_json::to_string(&e).unwrap_or_else(|_| e.to_string())))?;
 
-    Ok(PyValue(result.to_value()).to_object(py))
+    PyValue(result.to_value()).into_py_any(py)
 }

bindings/python/src/lib.rs

@@ -1,8 +1,9 @@
 use crate::decision::PyZenDecision;
 use crate::engine::PyZenEngine;
 use crate::expression::{evaluate_expression, evaluate_unary_expression, render_template};
+use pyo3::prelude::PyModuleMethods;
 use pyo3::types::PyModule;
-use pyo3::{pymodule, wrap_pyfunction, PyResult, Python};
+use pyo3::{pymodule, wrap_pyfunction, Bound, PyResult, Python};
 
 mod custom_node;
 mod decision;
@@ -13,7 +14,7 @@ mod types;
 mod value;
 
 #[pymodule]
-fn zen(_py: Python, m: &PyModule) -> PyResult<()> {
+fn zen(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyZenEngine>()?;
     m.add_class::<PyZenDecision>()?;
     m.add_function(wrap_pyfunction!(evaluate_expression, m)?)?;

bindings/python/src/loader.rs

@@ -2,13 +2,13 @@ use std::future::Future;
 use std::sync::Arc;
 
 use anyhow::anyhow;
-use pyo3::{PyObject, Python};
+use pyo3::{Py, PyAny, PyObject, Python};
 
 use zen_engine::loader::{DecisionLoader, LoaderError, LoaderResponse};
 use zen_engine::model::DecisionContent;
 
 #[derive(Default)]
-pub(crate) struct PyDecisionLoader(Option<PyObject>);
+pub(crate) struct PyDecisionLoader(Option<Py<PyAny>>);
 
 impl From<PyObject> for PyDecisionLoader {
     fn from(value: PyObject) -> Self {
@@ -16,8 +16,8 @@ impl From<PyObject> for PyDecisionLoader {
     }
 }
 
-impl From<Option<PyObject>> for PyDecisionLoader {
-    fn from(value: Option<PyObject>) -> Self {
+impl From<Option<Py<PyAny>>> for PyDecisionLoader {
+    fn from(value: Option<Py<PyAny>>) -> Self {
         Self(value)
     }
 }

bindings/python/src/types.rs

@@ -1,6 +1,6 @@
 use anyhow::{anyhow, Context};
 use json_dotpath::DotPaths;
-use pyo3::{pyclass, pymethods, PyObject, PyResult, Python, ToPyObject};
+use pyo3::{pyclass, pymethods, IntoPyObjectExt, Py, PyAny, PyResult, Python};
 use serde::Serialize;
 use serde_json::Value;
 use std::sync::Arc;
@@ -37,9 +37,9 @@ pub struct PyNodeRequest {
     inner_input: Value,
 
     #[pyo3(get)]
-    pub input: PyObject,
+    pub input: Py<PyAny>,
     #[pyo3(get)]
-    pub node: PyObject,
+    pub node: Py<PyAny>,
 }
 
 impl PyNodeRequest {
@@ -48,8 +48,8 @@ impl PyNodeRequest {
         let node_val = serde_json::to_value(&inner_node).unwrap();
 
         Ok(Self {
-            input: value_to_object(py, &value.input.to_value()),
-            node: value_to_object(py, &node_val),
+            input: value_to_object(py, &value.input.to_value())?.unbind(),
+            node: value_to_object(py, &node_val)?.unbind(),
 
             inner_input: value.input.to_value(),
             inner_node,
@@ -59,7 +59,7 @@ impl PyNodeRequest {
 
 #[pymethods]
 impl PyNodeRequest {
-    fn get_field(&self, py: Python, path: String) -> PyResult<PyObject> {
+    fn get_field(&self, py: Python, path: String) -> PyResult<Py<PyAny>> {
         let node_config = &self.inner_node.config;
 
         let selected_value: Value = node_config
@@ -68,16 +68,16 @@ impl PyNodeRequest {
             .flatten()
             .context("Failed to find JSON path")?;
         let Value::String(template) = selected_value else {
-            return Ok(PyValue(selected_value).to_object(py));
+            return PyValue(selected_value).into_py_any(py);
         };
 
         let template_value = zen_tmpl::render(template.as_str(), Variable::from(&self.inner_input))
             .map_err(|e| anyhow!(serde_json::to_string(&e).unwrap_or_else(|_| e.to_string())))?;
 
-        Ok(PyValue(template_value.to_value()).to_object(py))
+        PyValue(template_value.to_value()).into_py_any(py)
     }
 
-    fn get_field_raw(&self, py: Python, path: String) -> PyResult<PyObject> {
+    fn get_field_raw(&self, py: Python, path: String) -> PyResult<Py<PyAny>> {
         let node_config = &self.inner_node.config;
 
         let selected_value: Value = node_config
@@ -86,7 +86,7 @@ impl PyNodeRequest {
             .flatten()
             .context("Failed to find JSON path")?;
 
-        Ok(PyValue(selected_value).to_object(py))
+        PyValue(selected_value).into_py_any(py)
     }
 }
 

bindings/python/src/value.rs

@@ -1,35 +1,48 @@
-use pyo3::{PyObject, Python, ToPyObject};
+use pyo3::prelude::{PyDictMethods, PyListMethods};
+use pyo3::types::{PyDict, PyList};
+use pyo3::{Bound, IntoPyObject, IntoPyObjectExt, PyAny, PyErr, PyResult, Python};
 use serde_json::Value;
-use std::collections::HashMap;
 
 #[repr(transparent)]
 #[derive(Clone, Debug)]
 pub struct PyValue(pub Value);
 
-pub fn value_to_object(py: Python<'_>, val: &Value) -> PyObject {
+pub fn value_to_object<'py>(py: Python<'py>, val: &Value) -> PyResult<Bound<'py, PyAny>> {
     match val {
-        Value::Null => py.None(),
-        Value::Bool(b) => b.to_object(py),
+        Value::Null => py.None().into_bound_py_any(py),
+        Value::Bool(b) => b.into_bound_py_any(py),
         Value::Number(n) => {
-            let oi64 = n.as_i64().map(|i| i.to_object(py));
-            let ou64 = n.as_u64().map(|i| i.to_object(py));
-            let of64 = n.as_f64().map(|i| i.to_object(py));
+            let oi64 = n.as_i64().map(|i| i.into_bound_py_any(py));
+            let ou64 = n.as_u64().map(|i| i.into_bound_py_any(py));
+            let of64 = n.as_f64().map(|i| i.into_bound_py_any(py));
             oi64.or(ou64).or(of64).expect("number too large")
         }
-        Value::String(s) => s.to_object(py),
+        Value::String(s) => s.into_bound_py_any(py),
         Value::Array(v) => {
-            let inner: Vec<_> = v.iter().map(|x| value_to_object(py, x)).collect();
-            inner.to_object(py)
+            let list = PyList::empty(py);
+            for item in v {
+                list.append(value_to_object(py, item)?)?;
+            }
+
+            list.into_bound_py_any(py)
         }
         Value::Object(m) => {
-            let inner: HashMap<_, _> = m.iter().map(|(k, v)| (k, value_to_object(py, v))).collect();
-            inner.to_object(py)
+            let dict = PyDict::new(py);
+            for (key, value) in m {
+                dict.set_item(key, value_to_object(py, value)?)?;
+            }
+
+            dict.into_bound_py_any(py)
         }
     }
 }
 
-impl ToPyObject for PyValue {
-    fn to_object(&self, py: Python<'_>) -> PyObject {
+impl<'py> IntoPyObject<'py> for PyValue {
+    type Target = PyAny;
+    type Output = Bound<'py, PyAny>;
+    type Error = PyErr;
+
+    fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
         value_to_object(py, &self.0)
     }
 }