refactor pyclass initialization

guide/src/migration.md

@@ -59,6 +59,35 @@ let _: Bound<'_, PyNone> = unsafe { Bound::from_owned_ptr(py, raw_ptr).cast_into
 # })
 ```
 
+### Removal of `From<Bound<'_, T>` and `From<Py<T>> for PyClassInitializer<T>`
+
+As part of refactoring the initialization code these impls were removed and its functionality was moved into the generated code for `#[new]`.
+As a small side side effect the following pattern will not be accepted anymore:
+
+```rust,ignore
+# use pyo3::prelude::*;
+# Python::attach(|py| {
+# let existing_py: Py<PyAny> = py.None();
+let obj_1 = Py::new(py, existing_py);
+
+# let existing_bound: Bound<'_, PyAny> = py.None().into_bound(py);
+let obj_2 = Bound::new(py, existing_bound);
+# })
+```
+
+To migrate use `clone` or `clone_ref`:
+
+```rust
+# use pyo3::prelude::*;
+# Python::attach(|py| {
+# let existing_py: Py<PyAny> = py.None();
+let obj_1 = existing_py.clone_ref(py);
+
+# let existing_bound: Bound<'_, PyAny> = py.None().into_bound(py);
+let obj_2 = existing_bound.clone();
+# })
+```
+
 ### Internal change to use multi-phase initialization
 
 [PEP 489](https://peps.python.org/pep-0489/) introduced "multi-phase initialization" for extension modules which provides ways to allocate and clean up per-module state.

newsfragments/5739.changed.md

@@ -0,0 +1 @@
+`#[new]` can now return arbitrary Python objects

pyo3-macros-backend/src/method.rs

@@ -427,7 +427,6 @@ pub struct FnSpec<'a> {
     pub asyncness: Option<syn::Token![async]>,
     pub unsafety: Option<syn::Token![unsafe]>,
     pub warnings: Vec<PyFunctionWarning>,
-    #[cfg(feature = "experimental-inspect")]
     pub output: syn::ReturnType,
 }
 
@@ -503,7 +502,6 @@ impl<'a> FnSpec<'a> {
             asyncness: sig.asyncness,
             unsafety: sig.unsafety,
             warnings,
-            #[cfg(feature = "experimental-inspect")]
             output: sig.output.clone(),
         })
     }

pyo3-macros-backend/src/pyclass.rs

@@ -1817,7 +1817,6 @@ fn complex_enum_struct_variant_new<'a>(
         asyncness: None,
         unsafety: None,
         warnings: vec![],
-        #[cfg(feature = "experimental-inspect")]
         output: syn::ReturnType::Default,
     };
 
@@ -1875,7 +1874,6 @@ fn complex_enum_tuple_variant_new<'a>(
         asyncness: None,
         unsafety: None,
         warnings: vec![],
-        #[cfg(feature = "experimental-inspect")]
         output: syn::ReturnType::Default,
     };
 
@@ -1903,7 +1901,6 @@ fn complex_enum_variant_field_getter<'a>(
         asyncness: None,
         unsafety: None,
         warnings: vec![],
-        #[cfg(feature = "experimental-inspect")]
         output: parse_quote!(-> #variant_cls_type),
     };
 

pyo3-macros-backend/src/pyfunction.rs

@@ -415,7 +415,6 @@ pub fn impl_wrap_pyfunction(
         asyncness: func.sig.asyncness,
         unsafety: func.sig.unsafety,
         warnings,
-        #[cfg(feature = "experimental-inspect")]
         output: func.sig.output.clone(),
     };
 

pyo3-macros-backend/src/pymethod.rs

@@ -16,8 +16,8 @@ use crate::{
 use crate::{quotes, utils};
 use proc_macro2::{Span, TokenStream};
 use quote::{format_ident, quote, quote_spanned, ToTokens};
-use syn::LitCStr;
 use syn::{ext::IdentExt, spanned::Spanned, Field, Ident, Result};
+use syn::{parse_quote, LitCStr};
 
 /// Generated code for a single pymethod item.
 pub struct MethodAndMethodDef {
@@ -1460,15 +1460,23 @@ fn generate_method_body(
                 }
             });
 
+            let output = if let syn::ReturnType::Type(_, ty) = &spec.output {
+                ty
+            } else {
+                &parse_quote!(())
+            };
             let body = quote! {
                 #text_signature_impl
 
-                use #pyo3_path::impl_::callback::IntoPyCallbackOutput;
+                use #pyo3_path::impl_::pyclass::Probe as _;
                 #warnings
                 #arg_convert
                 let result = #call;
-                let initializer: #pyo3_path::PyClassInitializer::<#cls> = result.convert(py)?;
-                #pyo3_path::impl_::pymethods::tp_new_impl(py, initializer, _slf)
+                #pyo3_path::impl_::pymethods::tp_new_impl::<
+                    _,
+                    { #pyo3_path::impl_::pyclass::IsPyClass::<#output>::VALUE },
+                    { #pyo3_path::impl_::pyclass::IsInitializerTuple::<#output>::VALUE }
+                >(py, result, _slf)
             };
             (arg_idents, arg_types, body)
         }

src/impl_/pyclass/probes.rs

@@ -1,6 +1,9 @@
 use std::marker::PhantomData;
 
-use crate::{conversion::IntoPyObject, FromPyObject, Py};
+use crate::conversion::IntoPyObject;
+use crate::impl_::pyclass::PyClassBaseType;
+use crate::impl_::pyclass_init::PyNativeTypeInitializer;
+use crate::{FromPyObject, Py, PyClass, PyClassInitializer};
 
 /// Trait used to combine with zero-sized types to calculate at compile time
 /// some property of a type.
@@ -89,6 +92,39 @@ impl<E> IsReturningEmptyTuple<Result<(), E>> {
     pub const VALUE: bool = true;
 }
 
+probe!(IsPyClass);
+impl<T> IsPyClass<T>
+where
+    T: PyClass,
+{
+    pub const VALUE: bool = true;
+}
+
+impl<T, E> IsPyClass<Result<T, E>>
+where
+    T: PyClass,
+{
+    pub const VALUE: bool = true;
+}
+
+probe!(IsInitializerTuple);
+impl<S, B> IsInitializerTuple<(S, B)>
+where
+    S: PyClass<BaseType = B>,
+    B: PyClass + PyClassBaseType<Initializer = PyClassInitializer<B>>,
+    B::BaseType: PyClassBaseType<Initializer = PyNativeTypeInitializer<B::BaseType>>,
+{
+    pub const VALUE: bool = true;
+}
+impl<S, B, E> IsInitializerTuple<Result<(S, B), E>>
+where
+    S: PyClass<BaseType = B>,
+    B: PyClass + PyClassBaseType<Initializer = PyClassInitializer<B>>,
+    B::BaseType: PyClassBaseType<Initializer = PyNativeTypeInitializer<B::BaseType>>,
+{
+    pub const VALUE: bool = true;
+}
+
 #[cfg(test)]
 macro_rules! value_of {
     ($probe:ident, $ty:ty) => {{

src/impl_/pyclass_init.rs

@@ -1,9 +1,10 @@
 //! Contains initialization utilities for `#[pyclass]`.
 use crate::exceptions::PyTypeError;
 use crate::ffi_ptr_ext::FfiPtrExt;
+use crate::impl_::pyclass::PyClassBaseType;
 use crate::internal::get_slot::TP_NEW;
 use crate::types::{PyTuple, PyType};
-use crate::{ffi, PyErr, PyResult, Python};
+use crate::{ffi, PyClass, PyClassInitializer, PyErr, PyResult, Python};
 use crate::{ffi::PyTypeObject, sealed::Sealed, type_object::PyTypeInfo};
 use std::marker::PhantomData;
 
@@ -19,9 +20,6 @@ pub trait PyObjectInit<T>: Sized + Sealed {
         py: Python<'_>,
         subtype: *mut PyTypeObject,
     ) -> PyResult<*mut ffi::PyObject>;
-
-    #[doc(hidden)]
-    fn can_be_subclassed(&self) -> bool;
 }
 
 /// Initializer for Python native types, like `PyDict`.
@@ -57,9 +55,84 @@ impl<T: PyTypeInfo> PyObjectInit<T> for PyNativeTypeInitializer<T> {
         }
         unsafe { inner(py, T::type_object_raw(py), subtype) }
     }
+}
+
+pub trait PyClassInit<'py, const IS_PYCLASS: bool, const IS_INITIALIZER_TUPLE: bool> {
+    fn init(
+        self,
+        cls: crate::Borrowed<'_, 'py, crate::types::PyType>,
+    ) -> PyResult<crate::Bound<'py, crate::PyAny>>;
+}
+
+impl<'py, T> PyClassInit<'py, false, false> for T
+where
+    T: crate::IntoPyObject<'py>,
+{
+    fn init(
+        self,
+        cls: crate::Borrowed<'_, 'py, crate::types::PyType>,
+    ) -> PyResult<crate::Bound<'py, crate::PyAny>> {
+        self.into_pyobject(cls.py())
+            .map(crate::BoundObject::into_any)
+            .map(crate::BoundObject::into_bound)
+            .map_err(Into::into)
+    }
+}
 
-    #[inline]
-    fn can_be_subclassed(&self) -> bool {
-        true
+impl<'py, T> PyClassInit<'py, true, false> for T
+where
+    T: crate::PyClass,
+    T::BaseType:
+        super::pyclass::PyClassBaseType<Initializer = PyNativeTypeInitializer<T::BaseType>>,
+{
+    fn init(
+        self,
+        cls: crate::Borrowed<'_, 'py, crate::types::PyType>,
+    ) -> PyResult<crate::Bound<'py, crate::PyAny>> {
+        PyClassInitializer::from(self).init(cls)
+    }
+}
+
+impl<'py, T, E, const IS_PYCLASS: bool, const IS_INITIALIZER_TUPLE: bool>
+    PyClassInit<'py, IS_PYCLASS, IS_INITIALIZER_TUPLE> for Result<T, E>
+where
+    T: PyClassInit<'py, IS_PYCLASS, IS_INITIALIZER_TUPLE>,
+    E: Into<PyErr>,
+{
+    fn init(
+        self,
+        cls: crate::Borrowed<'_, 'py, crate::types::PyType>,
+    ) -> PyResult<crate::Bound<'py, crate::PyAny>> {
+        self.map_err(Into::into)?.init(cls)
+    }
+}
+
+impl<'py, T> PyClassInit<'py, false, false> for PyClassInitializer<T>
+where
+    T: PyClass,
+{
+    fn init(
+        self,
+        cls: crate::Borrowed<'_, 'py, crate::types::PyType>,
+    ) -> PyResult<crate::Bound<'py, crate::PyAny>> {
+        unsafe {
+            self.create_class_object_of_type(cls.py(), cls.as_ptr().cast())
+                .map(crate::Bound::into_any)
+        }
+    }
+}
+
+impl<'py, S, B> PyClassInit<'py, false, true> for (S, B)
+where
+    S: PyClass<BaseType = B>,
+    B: PyClass + PyClassBaseType<Initializer = PyClassInitializer<B>>,
+    B::BaseType: PyClassBaseType<Initializer = PyNativeTypeInitializer<B::BaseType>>,
+{
+    fn init(
+        self,
+        cls: crate::Borrowed<'_, 'py, crate::types::PyType>,
+    ) -> PyResult<crate::Bound<'py, crate::PyAny>> {
+        let (sub, base) = self;
+        PyClassInitializer::from(base).add_subclass(sub).init(cls)
     }
 }

src/impl_/pymethods.rs

@@ -9,8 +9,8 @@ use crate::pycell::{PyBorrowError, PyBorrowMutError};
 use crate::pyclass::boolean_struct::False;
 use crate::types::PyType;
 use crate::{
-    ffi, Bound, CastError, Py, PyAny, PyClass, PyClassGuard, PyClassGuardMut, PyClassInitializer,
-    PyErr, PyRef, PyRefMut, PyResult, PyTraverseError, PyTypeCheck, PyVisit, Python,
+    ffi, Bound, CastError, Py, PyAny, PyClass, PyClassGuard, PyClassGuardMut, PyErr, PyRef,
+    PyRefMut, PyResult, PyTraverseError, PyTypeCheck, PyVisit, Python,
 };
 use std::ffi::CStr;
 use std::ffi::{c_int, c_void};
@@ -724,14 +724,16 @@ impl<'py, T> std::ops::Deref for BoundRef<'_, 'py, T> {
     }
 }
 
-pub unsafe fn tp_new_impl<T: PyClass>(
-    py: Python<'_>,
-    initializer: PyClassInitializer<T>,
-    target_type: *mut ffi::PyTypeObject,
-) -> PyResult<*mut ffi::PyObject> {
+pub unsafe fn tp_new_impl<'py, T, const IS_PYCLASS: bool, const IS_INITIALIZER_TUPLE: bool>(
+    py: Python<'py>,
+    obj: T,
+    cls: *mut ffi::PyTypeObject,
+) -> PyResult<*mut ffi::PyObject>
+where
+    T: super::pyclass_init::PyClassInit<'py, IS_PYCLASS, IS_INITIALIZER_TUPLE>,
+{
     unsafe {
-        initializer
-            .create_class_object_of_type(py, target_type)
+        obj.init(crate::Borrowed::from_ptr_unchecked(py, cls.cast()).cast_unchecked())
             .map(Bound::into_ptr)
     }
 }