Vendor deps.

This commit is contained in:
jimmy committed 2026-09-16 20:10:35 -04:00
1 parent 294527085e
commit 91e18fde3a
1714 files changed
+556667 -7

No files matched your search

+19
View File
@@ -0,0 +1,19 @@
[book]
title = "PyO3 user guide"
description = "PyO3 user guide"
authors = ["PyO3 Project and Contributors"]
[preprocessor.pyo3_version]
command = "python3 pyo3_version.py"
[preprocessor.glossary_linker]
command = "python3 glossary_linker.py"
[preprocessor.tabs]
[output.html]
git-repository-url = "https://github.com/PyO3/pyo3/tree/main/guide"
edit-url-template = "https://github.com/PyO3/pyo3/edit/main/guide/{path}"
playground.runnable = false
additional-css = ["theme/tabs.css"]
additional-js = ["theme/tabs.js"]
+251
View File
@@ -0,0 +1,251 @@
"""mdbook preprocessor to auto-link glossary terms.
Parses glossary.md for terms defined using definition list syntax:
term A
: This is a definition of term A.
term B
: This is a definition of term B.
For each chapter (except the glossary itself), the first occurrence of each
term is replaced with a link to the glossary entry.
An HTML comment block at the bottom of glossary.md maps additional
terms to arbitrary URLs (e.g. upstream Python/Rust glossary entries):
Tested against mdbook 0.5.0.
"""
import json
import re
import sys
def slugify(term):
"""Convert a term to a URL-friendly anchor slug."""
return re.sub(r"[^\w\s-]", "", term.lower()).strip().replace(" ", "-")
def parse_glossary(content):
"""Parse glossary.md and return (local_terms, external_terms).
local_terms: dict of {term: url} for definition-list entries (link to glossary anchors).
external_terms: dict of {term: url} from the <!-- external-glossary-links --> comment.
"""
local_terms = {}
external_terms = {}
# Parse definition-list terms: a non-indented line followed by a line
# starting with " : " (the definition).
for m in re.finditer(r"(?m)^([^\s#<>!`\[].+)\n : ", content):
term = m.group(1).strip()
local_terms[term] = f"glossary.md#{slugify(term)}"
# Parse <!-- external-glossary-links ... --> block for external URLs.
link_block = re.search(
r"<!--\s*external-glossary-links\s*\n(.*?)-->", content, re.DOTALL
)
if not link_block:
raise ValueError("Glossary is missing <!-- external-glossary-links --> block")
for line in link_block.group(1).strip().splitlines():
line = line.strip()
if not line:
continue
# "term: url" or "term | url"
parts = re.split(r"\s*[:|]\s*", line, maxsplit=1)
if len(parts) == 2:
term, url = parts[0].strip(), parts[1].strip()
if term and url:
external_terms[term] = url
return local_terms, external_terms
# ---------------------------------------------------------------------------
# Replacement logic
# ---------------------------------------------------------------------------
# Matches fenced code blocks, inline code, and markdown links so we can skip
# them. Everything else is "plain text" we can scan for terms.
_SKIP_PATTERN = re.compile(
r"^```[^\n]*\n[\s\S]*?^```\s*$" # fenced code blocks (``` to ```, multiline)
r"|`[^`\n]+`" # inline code (single line only)
r"|\[[^\]]*\]\([^\)]*\)" # markdown links (entire [...](...))
r"|<[^>\n]+>" # HTML tags (single line only)
r"|^\s*>.*$" # block quotes (single line)
r"|^.+(?=\n : )" # definition list term lines
r"|^#{1,6}\s+.*$", # headings
re.MULTILINE,
)
def link_terms_in_content(content, terms, first_only=True, url_prefix=""):
"""Replace occurrences of glossary terms with markdown links.
If first_only is True (default), only the first occurrence of each term is
linked. If False, every occurrence is linked.
Skips code blocks, inline code, existing links, and HTML tags.
"""
linked = set()
# Build a combined pattern matching any term, longest first so that
# multi-word terms match before their sub-terms.
sorted_terms = sorted(terms.keys(), key=len, reverse=True)
if not sorted_terms:
return content
# Allow optional trailing "s" so plurals like "wheels" match "wheel".
escaped = [re.escape(t) + r"s?" for t in sorted_terms]
term_pattern = re.compile(r"\b(" + "|".join(escaped) + r")\b")
# Find all protected spans.
protected = []
for m in _SKIP_PATTERN.finditer(content):
protected.append((m.start(), m.end()))
# Parse <!-- no-glossary:term --> comments to suppress specific terms.
# Maps line number to set of suppressed term names (lowercased).
_suppressed_re = re.compile(r"<!--\s*no-glossary-link:([\w\s-]+?)\s*-->")
suppressed_at_line = {}
for m in _suppressed_re.finditer(content):
term_name = m.group(1).strip().lower()
# Find the next line after the comment.
next_line_start = content.find("\n", m.end())
if next_line_start == -1:
continue
next_line_start += 1
next_line_end = content.find("\n", next_line_start)
if next_line_end == -1:
next_line_end = len(content)
suppressed_at_line.setdefault((next_line_start, next_line_end), set()).add(
term_name
)
def in_protected(start, end):
for ps, pe in protected:
if start >= ps and end <= pe:
return True
return False
def is_suppressed(canonical, start):
"""Check if this term is suppressed by a <!-- no-glossary:term --> on the same line."""
for (ls, le), suppressed_terms in suppressed_at_line.items():
if ls <= start < le and canonical.lower() in suppressed_terms:
return True
return False
result = []
last = 0
for m in term_pattern.finditer(content):
matched_term = m.group(1)
# Find the canonical term. Try exact match, then case-insensitive,
# then strip trailing "s" for plural forms.
canonical = None
for candidate in (matched_term, matched_term.rstrip("s")):
if candidate in terms:
canonical = candidate
break
for t in terms:
if t.lower() == candidate.lower():
canonical = t
break
if canonical is not None:
break
if canonical is None:
continue
if first_only and canonical in linked:
continue
if in_protected(m.start(), m.end()):
continue
if is_suppressed(canonical, m.start()):
continue
linked.add(canonical)
result.append(content[last : m.start()])
url = terms[canonical]
if url_prefix and not url.startswith("http"):
url = url_prefix + url
result.append(f"[{matched_term}]({url})")
last = m.end()
result.append(content[last:])
return "".join(result)
# ---------------------------------------------------------------------------
# mdbook preprocessor interface
# ---------------------------------------------------------------------------
def find_glossary_content(items):
"""Walk the book items to find and return the glossary chapter content."""
for item in items:
if not isinstance(item, dict) or "Chapter" not in item:
continue
ch = item["Chapter"]
if ch.get("path") and ch["path"].endswith("glossary.md"):
return ch["content"]
result = find_glossary_content(ch.get("sub_items", []))
if result is not None:
return result
return None
def process_item(item, local_terms, external_terms):
"""Recursively process a book item, linking glossary terms."""
if not isinstance(item, dict) or "Chapter" not in item:
return
ch = item["Chapter"]
path = ch.get("path", "")
if path and path.endswith("glossary.md"):
# On the glossary page itself, link external terms (all occurrences).
ch["content"] = link_terms_in_content(
ch["content"], external_terms, first_only=False
)
elif path:
# Compute relative prefix for local glossary links based on depth.
prefix = "../" * path.count("/")
all_terms = {**local_terms, **external_terms}
ch["content"] = link_terms_in_content(
ch["content"], all_terms, url_prefix=prefix
)
for sub in ch.get("sub_items", []):
process_item(sub, local_terms, external_terms)
def main():
for line in sys.stdin:
if not line.strip():
continue
[context, book] = json.loads(line)
# Parse glossary terms from the glossary chapter.
glossary_content = find_glossary_content(book["items"])
if glossary_content is None:
# No glossary found; pass through unchanged.
json.dump(book, fp=sys.stdout)
return
local_terms, external_terms = parse_glossary(glossary_content)
# Process all chapters.
for item in book["items"]:
process_item(item, local_terms, external_terms)
json.dump(book, fp=sys.stdout)
return
if __name__ == "__main__":
main()
+53
View File
@@ -0,0 +1,53 @@
`#[pyclass]` can be used with the following parameters:
| Parameter | Description |
| :- | :- |
| `constructor` | This is currently only allowed on [variants of complex enums][params-constructor]. It allows customization of the generated class constructor for each variant. It uses the same syntax and supports the same options as the `signature` attribute of functions and methods. |
| <span style="white-space: pre">`crate = "some::path"`</span> | Path to import the `pyo3` crate, if it's not accessible at `::pyo3`. |
| `dict` | Gives instances of this class an empty `__dict__` to store custom attributes. |
| `eq` | Implements `__eq__` using the `PartialEq` implementation of the underlying Rust datatype. |
| `eq_int` | Implements `__eq__` using `__int__` for simple enums. |
| <span style="white-space: pre">`extends = BaseType`</span> | Use a custom baseclass. Defaults to [`PyAny`][params-1] |
| <span style="white-space: pre">`freelist = N`</span> | Implements a [free list][params-2] of size N. This can improve performance for types that are often created and deleted in quick succession. Profile your code to see whether `freelist` is right for you. |
| `from_py_object` | Implement `FromPyObject` for this pyclass. Requires the pyclass to be `Clone`. |
| <span style="white-space: pre">`frozen`</span> | Declares that your pyclass is immutable. It removes the borrow checker overhead when retrieving a shared reference to the Rust struct, but disables the ability to get a mutable reference. |
| `generic` | Implements runtime parametrization for the class following [PEP 560](https://peps.python.org/pep-0560/). |
| `get_all` | Generates getters for all fields of the pyclass. |
| `hash` | Implements `__hash__` using the `Hash` implementation of the underlying Rust datatype. *Requires `eq` and `frozen`* |
| `immutable_type` | Makes the type object immutable. Supported on 3.14+ with the `abi3` feature active, or 3.10+ otherwise. |
| `mapping` | Inform PyO3 that this class is a [`Mapping`][params-mapping], and so leave its implementation of sequence C-API slots empty. |
| <span style="white-space: pre">`module = "module_name"`</span> | Python code will see the class as being defined in this module. Defaults to `builtins`. |
| <span style="white-space: pre">`name = "python_name"`</span> | Sets the name that Python sees this class as. Defaults to the name of the Rust struct. |
| `ord` | Implements `__lt__`, `__gt__`, `__le__`, & `__ge__` using the `PartialOrd` implementation of the underlying Rust datatype. *Requires `eq`* |
| `rename_all = "renaming_rule"` | Applies renaming rules to every getters and setters of a struct, or every variants of an enum. Possible values are: "camelCase", "kebab-case", "lowercase", "PascalCase", "SCREAMING-KEBAB-CASE", "SCREAMING_SNAKE_CASE", "snake_case", "UPPERCASE". |
| `sequence` | Inform PyO3 that this class is a [`Sequence`][params-sequence], and so leave its C-API mapping length slot empty. |
| `set_all` | Generates setters for all fields of the pyclass. |
| `new = "from_fields"` | Generates a default `__new__` constructor with all fields as parameters in the `new()` method. |
| `skip_from_py_object` | Prevents this PyClass from participating in the `FromPyObject: PyClass + Clone` blanket implementation. This allows a custom `FromPyObject` impl, even if `self` is `Clone`. |
| `str` | Implements `__str__` using the `Display` implementation of the underlying Rust datatype or by passing an optional format string `str="<format string>"`. *Note: The optional format string is only allowed for structs. `name` and `rename_all` are incompatible with the optional format string. Additional details can be found in the discussion on this [PR](https://github.com/PyO3/pyo3/pull/4233).* |
| `subclass` | Allows other Python classes and `#[pyclass]` to inherit from this class. Enums cannot be subclassed. |
| `unsendable` | Required if your struct is not [`Send`][params-3]. Rather than using `unsendable`, consider implementing your struct in a thread-safe way by e.g. substituting [`Rc`][params-4] with [`Arc`][params-5]. By using `unsendable`, your class will panic when accessed by another thread. Also note the Python's GC is multi-threaded and while unsendable classes will not be traversed on foreign threads to avoid UB, this can lead to memory leaks. |
| `weakref` | Allows this class to be [weakly referenceable][params-6]. |
All of these parameters can either be passed directly on the `#[pyclass(...)]` annotation, or as one or more accompanying `#[pyo3(...)]` annotations, e.g.:
```rust,ignore
// Argument supplied directly to the `#[pyclass]` annotation.
#[pyclass(name = "SomeName", subclass)]
struct MyClass {}
// Argument supplied as a separate annotation.
#[pyclass]
#[pyo3(name = "SomeName", subclass)]
struct MyClass {}
```
[params-1]: https://docs.rs/pyo3/latest/pyo3/types/struct.PyAny.html
[params-2]: https://en.wikipedia.org/wiki/Free_list
[params-3]: https://doc.rust-lang.org/std/marker/trait.Send.html
[params-4]: https://doc.rust-lang.org/std/rc/struct.Rc.html
[params-5]: https://doc.rust-lang.org/std/sync/struct.Arc.html
[params-6]: https://docs.python.org/3/library/weakref.html
[params-constructor]: https://pyo3.rs/latest/class.html#complex-enums
[params-mapping]: https://pyo3.rs/latest/class/protocols.html#mapping--sequence-types
[params-sequence]: https://pyo3.rs/latest/class/protocols.html#mapping--sequence-types
+53
View File
@@ -0,0 +1,53 @@
"""Simple mdbook preprocessor to inject pyo3 version into the guide.
It will replace:
- {{#PYO3_VERSION_TAG}} with the contents of the PYO3_VERSION_TAG environment var
- {{#PYO3_DOCS_URL}} with the location of docs (e.g. 'https://docs.rs/pyo3/0.13.2')
- {{#PYO3_CRATE_VERSION}} with a relevant toml snippet (e.g. 'version = "0.13.2"')
Tested against mdbook 0.5.0.
"""
import json
import os
import sys
# Set PYO3_VERSION in CI to build the correct version into links
PYO3_VERSION_TAG = os.environ.get("PYO3_VERSION_TAG", "main")
if PYO3_VERSION_TAG == "main":
PYO3_DOCS_URL = "https://pyo3.rs/main/doc"
PYO3_DOCS_VERSION = "latest"
PYO3_CRATE_VERSION = 'git = "https://github.com/pyo3/pyo3"'
else:
# v0.13.2 -> 0.13.2
version = PYO3_VERSION_TAG.lstrip("v")
PYO3_DOCS_URL = f"https://docs.rs/pyo3/{version}"
PYO3_DOCS_VERSION = version
PYO3_CRATE_VERSION = f'version = "{version}"'
def replace_item_content(item):
if not isinstance(item, dict) or "Chapter" not in item:
return
# Replace raw and url-encoded forms
item["Chapter"]["content"] = (
item["Chapter"]["content"]
.replace("{{#PYO3_VERSION_TAG}}", PYO3_VERSION_TAG)
.replace("{{#PYO3_DOCS_URL}}", PYO3_DOCS_URL)
.replace("{{#PYO3_DOCS_VERSION}}", PYO3_DOCS_VERSION)
.replace("{{#PYO3_CRATE_VERSION}}", PYO3_CRATE_VERSION)
)
for sub_item in item["Chapter"]["sub_items"]:
replace_item_content(sub_item)
for line in sys.stdin:
if line:
[context, book] = json.loads(line)
for item in book["items"]:
replace_item_content(item)
json.dump(book, fp=sys.stdout)
+58
View File
@@ -0,0 +1,58 @@
# Summary
[Introduction](index.md)
---
- [Getting started](getting-started.md)
- [Using Rust from Python](rust-from-python.md)
- [Python modules](module.md)
- [Python functions](function.md)
- [Function signatures](function/signature.md)
- [Error handling](function/error-handling.md)
- [Python classes](class.md)
- [Class customizations](class/protocols.md)
- [Basic object customization](class/object.md)
- [Emulating numeric types](class/numeric.md)
- [Emulating callable objects](class/call.md)
- [Thread safety](class/thread-safety.md)
- [Calling Python from Rust](python-from-rust.md)
- [Python object types](types.md)
- [Python exceptions](exception.md)
- [Calling Python functions](python-from-rust/function-calls.md)
- [Executing existing Python code](python-from-rust/calling-existing-code.md)
- [Type conversions](conversions.md)
- [Mapping of Rust types to Python types](conversions/tables.md)
- [Conversion traits](conversions/traits.md)
- [Using `async` and `await`](async-await.md)
- [Parallelism](parallelism.md)
- [Supporting Free-Threaded Python](free-threading.md)
- [Debugging](debugging.md)
- [Features reference](features.md)
- [Performance](performance.md)
- [Type stub generation and introspection](type-stub.md)
- [Advanced topics](advanced.md)
- [Sharing types between packages](advanced/sharing-types.md)
- [Building and distribution](building-and-distribution.md)
- [Supporting multiple Python versions](building-and-distribution/multiple-python-versions.md)
- [Useful crates](ecosystem.md)
- [Logging](ecosystem/logging.md)
- [Tracing](ecosystem/tracing.md)
- [Using `async` and `await`](ecosystem/async-await.md)
- [FAQ and troubleshooting](faq.md)
---
[Appendix A: Migration guide](migration.md)
[Appendix B: Glossary](glossary.md)
[Appendix B: Trait bounds](trait-bounds.md)
[Appendix C: Python typing hints](python-typing-hints.md)
[CHANGELOG](changelog.md)
---
[Contributing](contributing.md)
+14
View File
@@ -0,0 +1,14 @@
# Advanced topics
## FFI
PyO3 exposes much of Python's C API through the `ffi` module.
The C API is naturally unsafe and requires you to manage reference counts, errors and specific invariants yourself.
Please refer to the [C API Reference Manual](https://docs.python.org/3/c-api/) and [The Rustonomicon](https://doc.rust-lang.org/nightly/nomicon/ffi.html) before using any function from that API.
## Sharing types between multiple PyO3 modules
It is possible (but complicated) to share types between multiple PyO3 packages which can be compiled and installed separately.
This allows for ecosystems of functionality to be built similar to the many packages built on top of NumPy.
While PyO3 does not yet have any built-in support for doing this, the next sub-chapter of this guide describes the general approach to doing this, as well as the safety limitations to be aware of.
+454
View File
@@ -0,0 +1,454 @@
# Sharing `#[pyclass]` types between multiple PyO3 extension modules
> [!WARNING]
> This is an advanced topic which requires reaching deeply into `unsafe` code.
> PyO3 does not have a stable API for doing this, and the approach documented here may break when updating to newer versions of PyO3.
> See [issue #1444](https://github.com/pyo3/pyo3/issues/1444) for ongoing discussion about this topic.
Some Python extension modules such as NumPy expose a C API which can be consumed by other extension modules to build functionality which directly exchanges native data without needing to go via Python objects.
This allows for higher performance than continually moving data in and out of Python objects.
It is a common request for PyO3 extension modules to be able to share `#[pyclass]` types (and other native data) across multiple crate/package boundaries in a similar fashion to NumPy.
Because PyO3 extension modules are each compiled as individual `cdylib` binaries, they cannot depend on each other in the typical Rust way of adding a Cargo dependency (which typically just includes the dependency statically inside the final compiled binary).
Instead, the correct way to share `#[pyclass]` data between separate PyO3 extension modules is to use `#[repr(C)]` Rust types and an FFI-like API to exchange data across the `cdylib` boundaries.
The solution in this subchapter will explain how to do this correctly.
## Quick summary
The solution which this subchapter describes shares types across extension module boundaries by defining three Rust crates (see also the [example project] on GitHub).
1. A crate `base-package-core` contains:
1. `#[repr(C)]` Rust types and APIs to be shared with downstream extension modules (not `#[pyclass]` types or `#[pymethods]`).
2. An API struct which contains function pointers to the API functions, version information, and any additional data.
3. A global static variable which stores the API struct, and a function to initialise this at runtime by importing from the `base_package` Python package.
The implementations of the APIs defined in step 1 will delegate to the function pointers in the API struct.
2. A crate `base-package` which is a PyO3 project containing `#[pyclass]` thin wrappers around the types exposed in `base-package-core`.
1. Implement the API functionality defined in `base-package-core` and populate the API struct accordingly.
2. As part of the Python module, export the API struct inside a Python `capsule` object.
The initialisation function defined in `base-package-core` will import this capsule and populate the global static copy of the API struct backing the shared APIs.
3. A crate `derived-package` which is a PyO3 project depending on `base-package-core` (not `base-package`).
1. As part of the Python module, import the API struct from the `capule` exposed by the `base_package` Python package and store it in `derived-package`'s copy of the global static variable defined in `base-package-core`.
2. Use the APIs from `base-package-core` to implement functionality which directly uses the shared types.
The sections below go into more detail about how to implement various parts of this solution and why the solution is architected this way.
## Technical background & limitations
The Python ecosystem provides a well-established mechanism for sharing C APIs between native extension modules using [Python `capsule` objects](#capsules).
This avoids traditional "dynamic linking" between native extension modules, which would rely on system-specific behavior to locate and load `base_package` during import of `derived_package` and to resolve any errors in version mismatch between the two packages.
The `capsule` mechanism works as follows:
1. `base_package` defines a `#[repr(C)]` "API" struct which is exported in a Python `capsule` at runtime.
2. `derived_package` delegates to Python's extension loading mechanism to locate `base_package` and load the API from the capsule.
3. `derived_package` then contains its own logic to [check a compatible version](#api-versioning) of `base_package` was loaded; this is necessary to ensure safe exchange of the API struct.
When sharing types between multiple PyO3 extension modules through a `capsule`, the complexity arises from two main sources:
- Each Rust extension module may be built with completely separate Rust toolchains and build settings.
- This means anything which is implementation-defined, such as the layout of `#[repr(Rust)] struct`s, the implementation of `std`, and even optimizations, might disagree between the two extension modules.
- Each Rust extension module contains a full statically-linked copy of its own dependencies.
- Any `static` global variables which are compiled in the `base-package` will have a **completely independent** copy in the `derived-package`.
This includes all `std` globals such as the [global allocator] and [panic hook].
- Any dependency version mismatches might mean that bugs in dependencies of `base-package` may not reproduce in the copy in `derived-package`, (e.g. if the common dependency `base-package-core` depends on `foo` 0.1, it is possible `base-package` will compile with `foo` 0.1.1 and `base-package-core` will compile with `foo` 0.1.2).
Practically speaking, this introduces the following limitations on extension modules wanting to share data in this way:
- Extensions must take extreme care to ensure that only `#[repr(C)]` types are shared across the package boundary.
In particular the default `#[repr(Rust)]` layout has no stability guarantee; _two extension modules sharing a `#[repr(Rust)]` data type is undefined behavior_.
It is also very easy to accidentally share `#[repr(Rust)]` types, see the [safety note on the `PyCapsule` type documentation]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyCapsule.html) for cases to consider when sharing data.
> [!WARNING] > Beware that PyO3's error type, `PyErr`, is not `#[repr(C)]` and cannot be shared across the package boundary > This is an easy mistake to make when exposing fallible APIs which cross the boundary. > There is a [later section on error handling](#error-handling) which suggests alternative strategies
- APIs which rely on global variables will not work as expected across the package boundary.
For example:
- The `#[global_allocator]` used by each extension will likely be different - each will need to ensure that any allocations are freed by the same allocator.
- The `std::io` locks (e.g. for `stdout`) will not be shared, so concurrent output from the two extensions may interleave in unexpected ways.
- PyO3 currently stores `#[pyclass]` types as global variables in static storage in each `cdylib` crate which compiles them.
This means that directly sharing a `#[pyclass]` type across multiple `cdylib` crates will currently silently create multiple distinct Python types.
To avoid this, the shared types cannot be `#[pyclass]`, instead the package exporting the type to Python needs to make private `#[pyclass]` which wraps the shared type.
PyO3 may remove this limitation in future.
## Using a capsule to create a shared API { #capsules }
Let's start with the example of NumPy as an extension which wants to offer a C API for other native extensions to consume.
The section above already established that Python native extensions can reuse data using the Python `capsule` type.
It is an opaque wrapper which can be used to exchange arbitrary native data between Python extension modules.
PyO3 provides the [`PyCapsule` type]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyCapsule.html) to create and consume capsules from Rust code.
NumPy creates a `capsule` which contains a pointer to the structure mapping the implementation of the ["NumPy C API"](https://numpy.org/doc/stable/reference/c-api/index.html).
While the [exact contents of this structure are generated](https://github.com/numpy/numpy/blob/40a1b0283beacb0d723f51e2046c8dc049da6eee/numpy/_core/code_generators/generate_numpy_api.py#L202-L210), the resulting API structure looks something like the following C code:
```C
// The API is defined as a fully type-erased array of "void pointers".
//
// This copy of the array is not public API, but internal to the NumPy implementation.
void* PyArray_API[] = {
// Some fields contain function pointers, type erased
(void *) PyArray_GetNDArrayCVersion,
// Some fields are empty
NULL,
// Some fields point to Python type objects
(void *) &PyArray_Type,
// ... the real API is a few hundred elements long. contents and length are version specific.
};
```
To consume this array from downstream C projects, NumPy also defines a C header file which uses C macros to define the downstream API in terms of cast indexing into this API structure:
```C
static void* PyArray_API[] = NULL;
// cast function pointers back to their correct type
#define PyArray_GetNDArrayCVersion (*(unsigned int (*)(void))PyArray_API[0])
// cast type objects back to `PyTypeObject *`
#define PyArray_Type (* (PyTypeObject *)PyArray_API[2])
// downstream packages must call this function before using any of the other APIS
static int PyArray_ImportNumPyAPI(void)
{
PyArray_API = /* ... */;
}
```
To expose an API from Rust code, we'll need to take a similar approach.
We have the choice of either matching NumPy and using an array of opaque pointers, or using a more typed API `struct` ([as long as it is `#[repr(C)]` - see the safety docs on the [`PyCapsule` type]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyCapsule.html)).
A typed `struct` helps to avoid mistakes in casting fields to the wrong type incorrectly, however additional care needs to be taken to ensure that the layout of the struct does not change incompatibly across non-breaking versions of the API.
This is discussed further in [the next section](#api-versioning).
The snippets below sketch out what the NumPy approach looks like in Rust, using either an array of opaque pointers or a typed API struct.
In both cases the public API functions are thin wrappers around the function pointers in the API struct, to provide ergonomics similar to the C macros in the NumPy example.
{{#tabs }}
{{#tab name="Using a pointer array" }}
If using a pointer array, `base-package-core` will define the pointer array, `BaseApi`, which contains function pointers and other data as opaque `*mut c_void` pointers:
```rust
// The number of fields in the API will grow over time as future
// versions add more APIs.
const BASE_API_ENTRIES: usize = 2;
// The pointer array itself
#[repr(transparent)]
pub struct BaseApi([*mut c_void; BASE_API_ENTRIES]);
// SAFETY: BaseApi never changes once loaded, so it will be shared between threads.
// (Manual implementations necessary due to `*mut c_void` not being `Send` or `Sync`).
//
// (This is likely not necessary if using the typed struct approach, as the compiler can see the real function pointers which are likely `Send` / `Sync`).
unsafe impl Send for BaseApi { }
unsafe impl Sync for BaseApi { }
/// Global variable which will be used by the API functions
static BASE_API: PyOnceLock<BaseApi> = PyOnceLock::new();
impl BaseApi {
/// Internal method used by the public methods to read pointers from the API
fn get(py: Python<'_>) -> &'static Self {
BASE_API.get(py).expect("`base_package` not yet imported")
}
}
/// Version identifiers for the API, returned by `get_api_version`.
#[repr(C)]
pub struct ApiVersion {
/* details omitted for now, see below section regarding API versioning */
}
/// Downstream packages must call this method to initialise the library API before
/// calling other functions.
pub fn import_base_package(py: Python<'_>) -> PyResult<()> {
BASE_API.get_or_try_init(py, || /* details of importing omitted for now, see future sections */)?;
}
// Public functions exported by `base_package-core` are thin wrappers around the function pointers
// in the API struct.
/// Returns the version of the `base_package` API loaded. `ApiVersion` is a `#[repr(C)]` struct
/// so can be safely shared across package boundaries.
#[inline]
pub fn get_api_version(py: Python<'_>) -> ApiVersion {
// SAFETY: BASE_API slot 0 is known to be the `get_api_version` function (`base_package` will set it).
let get_api_version: extern "C" fn() -> ApiVersion = unsafe { std::mem::transmute(BASE_API.get(py).0[0]) };
get_api_version()
}
```
{{#endtab }}
{{#tab name="Using a typed struct" }}
If using a typed struct, `base-package-core` will define `BaseApi` as a `#[repr(C)]` struct with typed fields:
```rust
// The API struct #[repr(C)] to ensure a stable layout across package boundaries.
// IMPORTANT: all fields must also be `repr(C)`, function pointers must be `extern "C"`,
// and fields can only be added to the end of the struct in non-breaking versions of the API.
#[repr(C)]
pub struct BaseApi {
get_api_version: extern "C" fn() -> ApiVersion,
}
/// Global variable which will be used by the API functions
static BASE_API: PyOnceLock<BaseApi> = PyOnceLock::new();
impl BaseApi {
/// Internal method used by the public methods to read pointers from the API
fn get(py: Python<'_>) -> &'static Self {
BASE_API.get(py).expect("`base_package` not yet imported")
}
}
/// Version identifiers for the API, returned by `get_api_version`.
#[repr(C)]
pub struct ApiVersion {
/* details omitted for now, see below section regarding API versioning */
}
/// Downstream packages must call this method to initialise the library API before
/// calling other functions.
pub fn import_base_package(py: Python<'_>) -> PyResult<()> {
BASE_API.get_or_try_init(py, || /* details of importing omitted for now, see future sections */)?;
}
// Public functions exported by `base_package-core` are thin wrappers around the function pointers
// in the API struct.
/// Returns the version of the `base_package` API loaded. `ApiVersion` is a `#[repr(C)]` struct
/// so can be safely shared across package boundaries.
#[inline]
pub fn get_api_version(py: Python<'_>) -> ApiVersion {
(BASE_API.get(py).get_api_version)()
}
```
{{#endtab }}
{{#endtabs }}
The consumers of the API struct will be compiled against a specific version of the `base-package-core` crate.
It will only be safe to use the API struct if the version in the API struct matches the version expected by the consumer.
Regardless of the choice made, due to backwards compatibility, the API struct can only grow over time, except when the version signals a breaking change.
The [example project] demonstrates how to do this with Rust.
## API versioning
To safely consume the API struct from downstream packages it is first necessary to perform a version check.
This version check needs to establish that the ABI (Application Binary Interface), i.e. the layout of the API struct, matches the expectations of the consumer.
This means that the `ApiVersion` type exposed by `base-package` in the example uses four fields: the three `major`, `minor`, and `patch` version fields from semver, plus an additional `abi_version` field which is only incremented for breaking changes to the ABI.
Having the `abi_version` field allows for consumers potentially be compatible even across semver-breaking versions of the API.
This means that e.g. `derived-package` compiled with version `0.0.3` of `base-package` could potentially be compatible with `base-package` version `0.0.2` if the API struct layout did not change between these versions, and the `abi_version` field was not incremented.
To make the version check straightforward, it is recommended to place the version information at the start of the API struct.
This allows the consumer to first read the capsule data as an `ApiVersion` structure, and only if the version check passes, reinterpret the rest of the data as the full API struct.
To demonstrate, the following code shows the approximate implementation of the `import_base_package` function from the [example project], which performs the version check and if successful reads the API struct from the capsule:
```rust
#[repr(C)]
pub struct BaseApi {
get_api_version: extern "C" fn() -> ApiVersion,
// real code will contain additional fields to satisfy all functionality
}
/// Version identifiers for the API, returned by `get_api_version`.
#[repr(C)]
pub struct ApiVersion {
major: u32,
minor: u32,
patch: u32,
abi: u32,
}
/// Downstream packages must call this method to initialise the library API before
/// calling other functions.
pub fn import_base_package(py: Python<'_>) -> PyResult<()> {
BASE_API.get_or_try_init(|| do_import(py))?;
}
fn do_import(py: Python<'_>) -> PyResult<BaseApi> {
// First: import the capsule as a pointer to retrieve version information. It is necessary to validate the API version
// before attempting to access any of the rest of the API.
//
// SAFETY: The function to get the version info is the first field in the API struct.
let capsule_base = unsafe { PyCapsule::import::<extern "C" fn() -> ApiVersion>(py, c"base_package._BASE_API")? };
// Read the version information via the function pointer.
let versions = (*capsule_base)();
// Use environment variables set by Cargo to validate the API is
// compatible with the version of the base package that is currently running.
let current_major: u32 = env!("CARGO_PKG_VERSION_MAJOR")
.parse()
.expect("invalid cargo package version");
let current_minor: u32 = env!("CARGO_PKG_VERSION_MINOR")
.parse()
.expect("invalid cargo package version");
// Critical: the ABI version must match exactly, otherwise the layout of the API struct
// is not known by this consumer
if versions.abi != BaseApi::CURRENT_ABI_VERSION {
return Err(PyErr::new::<pyo3::exceptions::PyImportError, _>(format!(
"base_package ABI version mismatch: expected {}, got {}",
BaseApi::CURRENT_ABI_VERSION,
versions.abi
)));
}
// In this example, the consumer allows for newer versions of the API to be used, as long
// as they had a compatible ABI version. Real projects may have different policies on breakage
// and forwards/backwards compatibility they are prepared to maintain.
if (versions.major, versions.minor) < (current_major, current_minor) {
return Err(PyErr::new::<pyo3::exceptions::PyImportError, _>(format!(
"base_package API version mismatch: expected at least {}.{}, got {}.{}",
env!("CARGO_PKG_VERSION_MAJOR"),
env!("CARGO_PKG_VERSION_MINOR"),
versions.major,
versions.minor
)));
}
// SAFETY: The version fields have been validated, so it is now known it
// is safe to cast the data to the known struct.
//
// The capsule contains a pointer to the full API struct, so this cast is sound.
let api = unsafe { NonNull::from_ref(capsule_base).cast::<BaseApi>().as_ref() };
Ok(BaseApi {
// all fields in the API are function pointers, so are copied trivially
..*api
})
}
```
## Creating a shared `#[pyclass]` type
Once the version checks are complete and the API struct loading is established, this technique can be expanded to share `#[pyclass]` types across the package boundary.
For a type named `SharedType`, the steps to achieve this are as follows:
1. `base-package-core` defines a `#[repr(C)]` struct which contains the data to be shared across the boundary.
<!-- rumdl-disable MD013 -->
2. The `BaseApi` struct defined in the previous sections is extended to include functions to manipulate this struct (these functions will later be provided by `base-package`).
At a minimum, this will probably include:
- `get_shared_type: extern "C" fn() -> Py<PyType>` - a function to get the `#[pyclass]` Python type object for `SharedType`.
- `create_shared_type: unsafe extern "C" fn(SharedType) -> Option<Py<SharedType>>` - a function to create a new instance of the `SharedType` struct and return it as a Python object.
This function is `unsafe` because the caller must ensure that the thread is attached to the interpreter (`Python<'py>` is a zero-sized type and not FFI-safe).
The return type on this function is wrapped in `Option` to allow for failure - see [the error handling section](#error-handling) for more details.
- `cast_shared_type: for<'a> extern "C" fn(Borrowed<'a, '_, SharedType>) -> &'a SharedType` - a function to extract a reference to the `SharedType` Rust struct from inside a Python object.
3. With the API struct extended to include these functions, the `base-package-core` crate can now implement PyO3 traits for `SharedType` in terms of those functions.
The crucial traits are:
- `PyTypeInfo` - the `get_type` function can delegate to the `get_shared_type` function pointer in the API struct.
- `IntoPyPyObject` - the `into_pyobject` function can delegate to the `create_shared_type` function pointer in the API struct.
- `FromPyObject<'_>` - the `extract` function can delegate to the `cast_shared_type` function pointer in the API struct.
<!-- rumdl-enable MD013 -->
4. `base-package` implements a `#[pyclass]` which is a thin wrapper around the `SharedType` struct, defining its Python functionality.
5. `base-package` implements the functions defined in the API struct to manipulate the `SharedType`, and populates the API struct with pointers to these functions as part of creating the `capsule`.
6. `derived-package` uses the existing `import_base_package()` function to load the API struct, and then can interact with it as a Python type via PyO3's smart pointers.
The [example project] demonstrates how to do this with a `Series` type implementing a "mini DataFrame API", to show how to use these stages to perform real work.
## Practical considerations for sharing data across the package boundary
As a reminder, there are two key restrictions to data which is shared across the package boundary:
- Only types with a stable layout, such as `#[repr(C)]` types, can be shared.
- Global variables are not shared across the boundary, and in particular for data sharing, the `#[global_allocator]` is not shared, so data allocated by one package must be freed by the same package.
This means that all Rust standard library types, such as `Vec`, `String`, and `Box`, cannot be shared across the boundary.
Even Rust tuples do not have a stable layout and cannot be shared.
There [is a proposal for a `#[repr(crabi)]` ABI](https://github.com/rust-lang/rust/issues/111423) which would define stable layouts for many Rust types, which would make it easier to share data.
This would still not solve types which contain heap allocations.
For now, a practical solution is to use the [`abi_stable`] crate, which provides many equivalents to Rust standard library types with stable internal layouts.
It uses vtables to allow for heap-allocated data to be shared, automatically ensuring the same allocator is used to free data as was used to allocate it.
This is the solution used in the [example project] to share `Vec`-like and `String`-like data across the boundary.
Using vtables introduces overhead (e.g. prevents inlining), however this is a necessary consequence of the limitations of sharing data across the package boundary.
### Error handling across the boundary { #error-handling }
Similar to many other types, PyO3's `PyErr` type is not currently `#[repr(C)]`, so cannot be shared across the package boundary.
The simplest approach to handling errors across the boundary is to use `Option` return types in the API struct, and to return `None` on error.
This is the approach taken in the [example project].
The downside of this approach is that `?` does not trivially work in the implementation of the API functions.
The suggested strategy is:
- Inside the API function implementations, convert `PyResult` to `Option` by using `PyErr::restore` to write the error to the Python thread state, and returning `None` on error.
- The wrappers in `base-package-core` which delegate to the API functions can then convert the `Option` back to `PyResult` by using `PyErr::fetch` to read the error from the Python thread state.
## Wrap-up, limitations & future work
This subchapter has attempted to detail how to use `capsule` objects to create an API for sharing Rust data between multiple PyO3 extension modules.
While complex, this is a workable technique for projects which need to have this functionality.
PyO3 does not yet offer any particular support for this use case and will likely be unable to provide a fully safe API to achieve this while data exchange is limited to `#[repr(C)]` types with pitfalls such as duplicate global variables.
Over time PyO3 might accumulate utilities for common pieces of this process.
Some places where this process process easier in the future include:
### Better `#[pyclass]` support
At the moment the `base-package-core` Rust API cannot use PyO3's `#[pyclass]` macro, because of the global variable backing the `#[pyclass]` implementation being a problem when duplicated into `base-package` and `derived-package`.
A possible solution is that PyO3 could have an option like `#[pyclass(shareable)]`, which could automatically generate the kind of `PyTypeInfo` / `IntoPyObject` / `FromPyObject` implementations which need to be hand-rolled at present.
To make this work, PyO3 would probably need to support [module state](https://github.com/PyO3/pyo3/pull/5600), and require the `base-package` which exports the shared type to manually initialize it and store it in module state.
There are many open design questions about how to make that work elegantly.
### `#[repr(C)]` error handling
At present, PyO3's `PyErr` type is a complex internal state machine which allows for lazy creation of Python exceptions.
This was convenient in early implementations of PyO3 but carried internal complexity and overhead.
PyO3's APIs have been trending in recent years towards allowing Rust code to use Rust error types for full control of overhead, e.g. the `IntoPyObject` and `FromPyObject` traits have a `type Error`, and the `#[pyfunction]` macros accept any `Result<T, E>` as long as the error type implements a conversion to `PyErr`.
It is likely that PyO3 will eventually transition the `PyErr` type to be a thin wrapper around `Py<PyBaseException>`, which would allow it to have a stable layout and participate in error handling across the boundary.
This primarily requires consideration about how to nudge existing dependents of `PyErr`'s "lazy" internals towards better practices.
### Better support for stable-layout types
The restriction of needing `#[repr(C)]` types to achieve a stable ABI for data sharing creates a lot of friction at the boundary.
It can also have implications for both efficiency and implementation of the `base-package-core` types.
The author's experience is that the Rust compiler doesn't yet have a mechanism to comprehensively lint against accidentally sharing types with unstable layouts across the boundary.
There may be value in an upstream effort to implement this so that projects using this `capsule` mechanism can avoid easy mistakes.
Outside of the Rust project itself, [`abi_stable`] crate is the most complete solution currently available to have a convenient way to correctly create a stable ABI.
However, `abi_stable` also appears to be largely unmaintained, so users wanting to depend on its functionality may need to consider reviving or forking it.
Furthermore, PyO3 currently doesn't implement `FromPyObject` or `IntoPyObject` conversions for `abi_stable` types.
This makes working with these types somewhat awkward / inefficient at the Python boundary.
PyO3 _could_ add an optional feature for this, however given the API surface involved it would be better to resolve the question of maintenance of `abi_stable` before adding such functionality to PyO3.
[global allocator]: https://doc.rust-lang.org/stable/std/alloc/trait.GlobalAlloc.html
[panic hook]: https://doc.rust-lang.org/stable/std/panic/fn.set_hook.html
[example project]: https://github.com/davidhewitt/pyo3-1444-prototype
[`abi_stable`]: https://github.com/rodrimati1992/abi_stable_crates/
+110
View File
@@ -0,0 +1,110 @@
# Using `async` and `await`
*This feature is still in active development.*
*See [the related issue](https://github.com/PyO3/pyo3/issues/1632).*
`#[pyfunction]` and `#[pymethods]` attributes also support `async fn`.
```rust,no_run
# #![allow(dead_code)]
# #[cfg(feature = "experimental-async")] {
use std::{thread, time::Duration};
use futures::channel::oneshot;
use pyo3::prelude::*;
#[pyfunction]
#[pyo3(signature=(seconds, result=None))]
async fn sleep(seconds: f64, result: Option<Py<PyAny>>) -> Option<Py<PyAny>> {
let (tx, rx) = oneshot::channel();
thread::spawn(move || {
thread::sleep(Duration::from_secs_f64(seconds));
tx.send(()).unwrap();
});
rx.await.unwrap();
result
}
# }
```
*Python awaitables instantiated with this method can only be awaited in `asyncio` context.*
*Other Python async runtime may be supported in the future.*
## `Send + 'static` constraint
Resulting future of an `async fn` decorated by `#[pyfunction]` must be `Send + 'static` to be embedded in a Python object.
As a consequence, `async fn` parameters and return types must also be `Send + 'static`, so it is not possible to have a signature like `async fn does_not_compile<'py>(arg: Bound<'py, PyAny>) -> Bound<'py, PyAny>`.
However, there is an exception for method receivers, so async methods can accept `&self`/ `&mut self`.
Note that this means that the class instance is borrowed for as long as the returned future is not completed, even across yield points and while waiting for I/O operations to complete.
Hence, other methods cannot obtain exclusive borrows while the future is still being polled.
This is the same as how async methods in Rust generally work but it is more problematic for Rust code interfacing with Python code due to pervasive shared mutability.
This strongly suggests to prefer shared borrows `&self` over exclusive ones `&mut self` to avoid racy borrow check failures at runtime.
## Implicitly attached to the interpreter
Even if it is not possible to pass a `py: Python<'py>` token to an `async fn`, we're still attached to the interpreter during the execution of the future – the same as for a regular `fn` without `Python<'py>`/`Bound<'py, PyAny>` parameter
It is still possible to get a `Python` marker using [`Python::attach`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.attach); because `attach` is reentrant and optimized, the cost will be negligible.
## Detaching from the interpreter across `.await`
There is currently no simple way to detach from the interpreter when awaiting a future, *but solutions are currently in development*.
Here is the advised workaround for now:
```rust,ignore
use std::{
future::Future,
pin::{Pin, pin},
task::{Context, Poll},
};
use pyo3::prelude::*;
struct AllowThreads<F>(F);
impl<F> Future for AllowThreads<F>
where
F: Future + Unpin + Send,
F::Output: Send,
{
type Output = F::Output;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let waker = cx.waker();
Python::attach(|py| {
py.detach(|| pin!(&mut self.0).poll(&mut Context::from_waker(waker)))
})
}
}
```
## Cancellation
Cancellation on the Python side can be caught using [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) type, by annotating a function parameter with `#[pyo3(cancel_handle)]`.
```rust,no_run
# #![allow(dead_code)]
# #[cfg(feature = "experimental-async")] {
use futures::FutureExt;
use pyo3::prelude::*;
use pyo3::coroutine::CancelHandle;
#[pyfunction]
async fn cancellable(#[pyo3(cancel_handle)] mut cancel: CancelHandle) {
futures::select! {
/* _ = ... => println!("done"), */
_ = cancel.cancelled().fuse() => println!("cancelled"),
}
}
# }
```
## The `Coroutine` type
To make a Rust future awaitable in Python, PyO3 defines a [`Coroutine`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.Coroutine.html) type, which implements the Python [coroutine protocol](https://docs.python.org/3/library/collections.abc.html#collections.abc.Coroutine).
Each `coroutine.send` call is translated to a `Future::poll` call.
If a [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) parameter is declared, the exception passed to `coroutine.throw` call is stored in it and can be retrieved with [`CancelHandle::cancelled`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html#method.cancelled); otherwise, it cancels the Rust future, and the exception is reraised;
*The type does not yet have a public constructor until the design is finalized.*
+448
View File
@@ -0,0 +1,448 @@
# Building and distribution
This chapter of the guide goes into detail on how to build and distribute projects using PyO3.
The way to achieve this is very different depending on whether the project is a Python module implemented in Rust, or a Rust binary embedding Python.
For both types of project there are also common problems such as the Python version to build for and the [linker](<https://en.wikipedia.org/wiki/Linker_(computing)>) arguments to use.
The material in this chapter is intended for users who have already read the PyO3 [README](./index.md).
It covers in turn the choices that can be made for Python modules and for Rust binaries.
There is also a section at the end about cross-compiling projects using PyO3.
There is an additional sub-chapter dedicated to [supporting multiple Python versions](./building-and-distribution/multiple-python-versions.md).
## Configuring the Python version
PyO3 uses a build script (backed by the [`pyo3-build-config`] crate) to determine the Python version and set the correct linker arguments.
By default it will attempt to use the following in order:
- Any active Python virtualenv.
- The `python` executable (if it's a Python 3 interpreter).
- The `python3` executable.
You can override the Python interpreter by setting the `PYO3_PYTHON` environment variable, e.g. `PYO3_PYTHON=python3.8`, `PYO3_PYTHON=/usr/bin/python3.9`, or even a PyPy interpreter `PYO3_PYTHON=pypy3`.
Once the Python interpreter is located, `pyo3-build-config` executes it to query the information in the `sysconfig` module which is needed to configure the rest of the compilation.
To validate the configuration which PyO3 will use, you can run a compilation with the environment variable `PYO3_PRINT_CONFIG=1` set.
An example output of doing this is shown below:
```console
$ PYO3_PRINT_CONFIG=1 cargo build
Compiling pyo3-ffi v0.29.2 (/Users/goldbaum/Documents/pyo3/pyo3-ffi)
error: failed to run custom build command for `pyo3-ffi v0.29.2 (/Users/goldbaum/Documents/pyo3/pyo3-ffi)`
Caused by:
process didn't exit successfully: `/Users/goldbaum/Documents/pyo3/target/debug/build/pyo3-ffi-71f0882ba738a1f0/build-script-build` (exit status: 101)
--- stdout
cargo:rustc-check-cfg=cfg(Py_LIMITED_API)
cargo:rustc-check-cfg=cfg(Py_GIL_DISABLED)
cargo:rustc-check-cfg=cfg(PyPy)
cargo:rustc-check-cfg=cfg(GraalPy)
cargo:rustc-check-cfg=cfg(RustPython)
cargo:rustc-check-cfg=cfg(py_sys_config, values("Py_DEBUG", "Py_REF_DEBUG", "Py_TRACE_REFS", "COUNT_ALLOCS"))
cargo:rustc-check-cfg=cfg(Py_3_8)
cargo:rustc-check-cfg=cfg(Py_3_9)
cargo:rustc-check-cfg=cfg(Py_3_10)
cargo:rustc-check-cfg=cfg(Py_3_11)
cargo:rustc-check-cfg=cfg(Py_3_12)
cargo:rustc-check-cfg=cfg(Py_3_13)
cargo:rustc-check-cfg=cfg(Py_3_14)
cargo:rustc-check-cfg=cfg(Py_3_15)
cargo:rustc-check-cfg=cfg(Py_3_16)
cargo:rerun-if-env-changed=PYO3_CONFIG_FILE
cargo:rerun-if-env-changed=PYO3_CROSS
cargo:rerun-if-env-changed=PYO3_CROSS_LIB_DIR
cargo:rerun-if-env-changed=PYO3_CROSS_PYTHON_VERSION
cargo:rerun-if-env-changed=PYO3_CROSS_PYTHON_IMPLEMENTATION
cargo:rerun-if-env-changed=PYO3_NO_PYTHON
cargo:rerun-if-env-changed=PYO3_ENVIRONMENT_SIGNATURE
cargo:rerun-if-env-changed=PYO3_PYTHON
cargo:rerun-if-env-changed=VIRTUAL_ENV
cargo:rerun-if-env-changed=CONDA_PREFIX
cargo:rerun-if-env-changed=PATH
cargo:rerun-if-env-changed=PYO3_PRINT_CONFIG
-- PYO3_PRINT_CONFIG=1 is set, printing configuration and halting compile --
implementation=CPython
version=3.14
shared=true
target_abi=CPython-free_threaded-3.14
lib_name=python3.14t
lib_dir=/Users/goldbaum/.pyenv/versions/3.14.5t/lib
executable=/Users/goldbaum/.pyenv/versions/3.14.5t/bin/python
pointer_width=64
build_flags=Py_GIL_DISABLED
python_framework_prefix=
suppress_build_script_link_lines=false
note: unset the PYO3_PRINT_CONFIG environment variable and retry to compile with the above config
```
The `PYO3_ENVIRONMENT_SIGNATURE` environment variable can be used to trigger rebuilds when its value changes, it has no other effect.
### Advanced: config files
If you save the above output config from `PYO3_PRINT_CONFIG` to a file, it is possible to manually override the contents and feed it back into PyO3 using the `PYO3_CONFIG_FILE` env var.
If your build environment is unusual enough that PyO3's regular configuration detection doesn't work, using a config file like this will give you the flexibility to make PyO3 work for you.
To see the full set of options supported, see the documentation for the [`InterpreterConfig` struct](https://docs.rs/pyo3-build-config/{{#PYO3_DOCS_VERSION}}/pyo3_build_config/struct.InterpreterConfig.html).
## Building Python extension modules
Python extension modules need to be compiled differently depending on the OS (and architecture) that they are being compiled for.
As well as multiple OSes (and architectures), there are also many different Python versions which are actively supported.
Packages uploaded to [PyPI](https://pypi.org/) usually want to upload prebuilt "wheels" covering many OS/arch/version combinations so that users on all these different platforms don't have to compile the package themselves.
Package vendors can opt-in to the "abi3" limited Python API which allows their wheels to be used on multiple Python versions, reducing the number of wheels they need to compile, but restricts the functionality they can use.
There are many ways to go about this: it is possible to use `cargo` to build the extension module (along with some manual work, which varies with OS).
The PyO3 ecosystem has two packaging tools, [`maturin`] and [`setuptools-rust`], which abstract over the OS difference and also support building wheels for PyPI upload.
PyO3 has some functionality for configuring projects when building Python extension modules:
- The `PYO3_BUILD_EXTENSION_MODULE` environment variable, which must be set when building Python extension modules. `maturin` and `setuptools-rust` set this automatically.
- The `abi3` and `abi3t` Cargo feature and their version-specific `abi3-pyXY` and `abi3t-pyXY` companions, which are used to opt-in to the limited Python API and a flavor of stable ABI in order to support multiple Python versions in a single wheel.
The free-threaded build of CPython cannot load `abi3` wheels but both builds can load `abi3t` wheels.
This section describes the packaging tools before describing how to build manually without them.
It then proceeds with an explanation of the `PYO3_BUILD_EXTENSION_MODULE` environment variable.
Finally, there is a section describing PyO3's `abi3` features.
### Packaging tools
The PyO3 ecosystem has two main choices to abstract the process of developing Python extension modules:
- [`maturin`] is a command-line tool to build, package and upload Python modules.
It makes opinionated choices about project layout meaning it needs very little configuration.
This makes it a great choice for users who are building a Python extension module from scratch and don't need flexibility.
- [`setuptools-rust`] is an add-on for `setuptools` which adds extra keyword arguments to the `setup.py` configuration file.
It requires more configuration than `maturin`, however this gives additional flexibility for users adding Rust to an existing Python package that can't satisfy `maturin`'s constraints.
Consult each project's documentation for full details on how to get started using them and how to upload wheels to PyPI.
It should be noted that while `maturin` is able to build [manylinux](https://github.com/pypa/manylinux)-compliant wheels out-of-the-box, `setuptools-rust` requires a bit more effort, [relying on Docker](https://setuptools-rust.readthedocs.io/en/latest/building_wheels.html) for this purpose.
There are also [`maturin-starter`] and [`setuptools-rust-starter`] examples in the PyO3 repository.
### Manual builds
To build a PyO3-based Python extension module manually, start by running `cargo build` as normal in a library project with the [`cdylib` crate type](https://doc.rust-lang.org/cargo/reference/cargo-targets.html#the-crate-type-field) while the `PYO3_BUILD_EXTENSION_MODULE` environment variable is set.
Once built, symlink (or copy) and rename the shared library from Cargo's `target/` directory to your desired output directory:
- on macOS, rename `libyour_module.dylib` to `your_module.so`.
- on Windows, rename `libyour_module.dll` to `your_module.pyd`.
- on Linux, rename `libyour_module.so` to `your_module.so`.
You can then open a Python shell in the output directory and you'll be able to run `import your_module`.
If you're packaging your library for redistribution, you should indicate the Python interpreter your library is compiled for by including the [platform tag](#platform-tags) in its name.
This prevents incompatible interpreters from trying to import your library.
If you're compiling for PyPy you _must_ include the platform tag, or PyPy will ignore the module.
#### Bazel builds
To use PyO3 with bazel one needs to manually configure PyO3, PyO3-ffi and PyO3-macros.
In particular, one needs to make sure that it is compiled with the right python flags for the version you intend to use.
For example see:
1. [github.com/abrisco/rules_pyo3](https://github.com/abrisco/rules_pyo3) -- General rules for building extension modules.
2. [github.com/OliverFM/pytorch_with_gazelle](https://github.com/OliverFM/pytorch_with_gazelle) -- for a minimal example of a repo that can use PyO3, PyTorch and Gazelle to generate python Build files.
3. [github.com/TheButlah/rules_pyo3](https://github.com/TheButlah/rules_pyo3) -- is somewhat dated.
#### Platform tags
Rather than using just the `.so` or `.pyd` extension suggested above (depending on OS), you can prefix the shared library extension with a platform tag to indicate the interpreter it is compatible with.
You can query your interpreter's platform tag from the `sysconfig` module.
Some example outputs of this are seen below:
```bash
# CPython 3.10 on macOS
.cpython-310-darwin.so
# PyPy 7.3 (Python 3.9) on Linux
$ python -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX"))'
.pypy39-pp73-x86_64-linux-gnu.so
```
So, for example, a valid module library name on CPython 3.10 for macOS is `your_module.cpython-310-darwin.so`, and its equivalent when compiled for PyPy 7.3 on Linux would be `your_module.pypy38-pp73-x86_64-linux-gnu.so`.
See [PEP 3149](https://peps.python.org/pep-3149/) for more background on platform tags.
#### macOS
On macOS, because the `PYO3_BUILD_EXTENSION_MODULE` environment variable disables linking to `libpython` ([see the next section](#the-extension-module-feature)), some additional linker arguments need to be set. `maturin` and `setuptools-rust` both pass these arguments for PyO3 automatically, but projects using manual builds will need to set these directly in order to support macOS.
The easiest way to set the correct linker arguments is to add a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) with the following content:
```rust,ignore
fn main() {
pyo3_build_config::add_extension_module_link_args();
}
```
Remember to also add `pyo3-build-config` to the `build-dependencies` section in `Cargo.toml`.
An alternative to using `pyo3-build-config` is add the following to a cargo configuration file (e.g. `.cargo/config.toml`):
```toml
[target.x86_64-apple-darwin]
rustflags = [
"-C", "link-arg=-undefined",
"-C", "link-arg=dynamic_lookup",
]
[target.aarch64-apple-darwin]
rustflags = [
"-C", "link-arg=-undefined",
"-C", "link-arg=dynamic_lookup",
]
```
Using the MacOS system python3 (`/usr/bin/python3`, as opposed to python installed via homebrew, pyenv, nix, etc.) may result in runtime errors such as `Library not loaded: @rpath/Python3.framework/Versions/3.8/Python3`.
The easiest way to set the correct linker arguments is to add a `build.rs` with the following content:
```rust,ignore
fn main() {
pyo3_build_config::add_python_framework_link_args();
}
```
Alternatively it can be resolved with another addition to `.cargo/config.toml`:
```toml
[build]
rustflags = [
"-C", "link-args=-Wl,-rpath,/Library/Developer/CommandLineTools/Library/Frameworks",
]
```
For more discussion on and workarounds for MacOS linking problems [see this issue](https://github.com/PyO3/pyo3/issues/1800#issuecomment-906786649).
Finally, don't forget that on MacOS the `extension-module` feature will cause `cargo test` to fail without the `--no-default-features` flag (see [the FAQ](https://pyo3.rs/main/faq.html#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror)).
### The `PYO3_BUILD_EXTENSION_MODULE` environment variable
<a name="the-extension-module-feature"></a> <!-- for backwards compatibility -->
By default PyO3 links to `libpython`.
This makes binaries, tests, and examples "just work".
However, Python extension modules on Unix must not link to libpython for [manylinux](https://www.python.org/dev/peps/pep-0513/) compliance.
The downside of not linking to `libpython` is that binaries, tests, and examples (which usually embed Python) will fail to build.
As a result, PyO3 uses an environment variable `PYO3_BUILD_EXTENSION_MODULE` to disable linking to `libpython`.
This should only be set when building a library for distribution.
`maturin >= 1.9.4` and `setuptools-rust >= 1.12` will set this for you automatically.
> [!NOTE]
> Historically PyO3 used an `extension-module` feature to perform the same function now done by the `PYO3_BUILD_EXTENSION_MODULE` env var.
> This feature caused linking to be disabled for all compile targets, including Rust tests and benchmarks.
>
> Projects are encouraged to migrate off the feature, as it caused [major development pain](faq.md#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror) due to the lack of linking.
### The `PYO3_USE_RAW_DYLIB` environment variable
When targeting Windows, PyO3 will attempt to use [`raw-dylib` linking](https://doc.rust-lang.org/reference/items/external-blocks.html#dylib-versus-raw-dylib) to avoid the need for users to provide an actual import library to link against.
On occasion the full Python import library may be needed (e.g. mixed C/Rust projects where the C code uses symbols not defined by `pyo3-ffi`).
In these cases, setting `PYO3_USE_RAW_DYLIB=0` can be used to disable `raw-dylib` linking.
> [!NOTE]
> Historically PyO3 used a `generate-import-lib` feature which needed external machinery to achieve the same result of `raw-dylib` linking.
### `Py_LIMITED_API`/`abi3`/`abi3t`
By default, Python extension modules can only be used with the same Python version they were compiled against.
For example, an extension module built for Python 3.5 can't be imported in Python 3.8.
[PEP 384](https://www.python.org/dev/peps/pep-0384/) introduced the idea of the limited Python API, which would have a stable ABI enabling extension modules built with it to be used against multiple Python versions.
The ABI defined by PEP 384 is called `abi3`, since it's stable for all Python 3.X releases.
In 2026, the steering council approved [PEP 803](https://www.python.org/dev/peps/pep-0803/)) which defines a new stable ABI, `abi3t`, that can be used on the free-threaded build of CPython.
Extensions built using the `abit3` ABI are importable on both the GIL-enabled and free-threaded builds of any CPython version newer than the target version.
So, `abi3t` extensions built for the Python 3.15 limited API will be importable on Python 3.15, 3.16, 3.17, and all future Python 3.X versions.
The advantage of building extension modules using the limited Python API is that package vendors only need to build and distribute a single copy (for each OS / architecture), and users can install it on all Python versions from the [minimum version](#minimum-python-version-for-abi3-and-abi3t-builds) and up.
The downside of this is that PyO3 can't use optimizations which rely on being compiled against a known exact Python version.
It's up to you to decide whether this matters for your extension module.
It's also possible to design your extension module such that you can distribute `abi3` or `abi3t` wheels but allow users compiling from source to benefit from additional optimizations - see the [support for multiple python versions](./building-and-distribution/multiple-python-versions.md) section of this guide, in particular the `#[cfg(Py_LIMITED_API)]` flag.
There are three steps involved in targeting `abi3` or `abi3t` when building Python packages as wheels:
1. Enable the `abi3` and/or `abi3t` feature in `pyo3`.
This ensures `pyo3` only calls Python C-API functions which are part of the stable API, and on Windows also ensures that the project links against the correct shared object (no special behavior is required on other platforms):
```toml
[dependencies]
pyo3 = { {{#PYO3_CRATE_VERSION}}, features = ["abi3", "abi3t"] }
```
Enabling both the `"abi3"` and `"abi3t"` features will produce an `abi3` extension if targeting Python 3.14 or older and an `abi3t` extension if targeting 3.15 or newer.
If you always want to produce `abi3t` wheels, you can enable just the `"abi3t"` feature, which will produce extensions targeting `abi3t` if the host interpreter is Python 3.15 or newer, but will produce version-specific extensions otherwise.
If you only enable the `"abi3"` feature, you will produce `abi3` extensions if the host interpreter is a GIL-enabled build of CPython and version-specific extensions otherwise.
We suggest enabling both features and using multiple python interpreters to build several wheels.
You can build an `abi3` wheel for your minimum supported Python version, a `cp314t` version-specific wheel using a free-threaded Python 3.14 interpreter, and an `abi3t` wheel using a Python 3.15 interpreter.
This will produce wheels targeting all versions of CPython that PyO3 supports.
2. Ensure that the built shared objects are correctly marked as `abi3` and `abi3t`.
This is accomplished by telling your build system that you're using the limited API.
[`maturin`] >= 0.9.0 and [`setuptools-rust`] >= 0.11.4 support `abi3` wheels and [`maturin`] >= 1.14 supports `abi3t` wheels.
3. Ensure that the `.whl` is correctly marked as `abi3` or `abi3t`.
For projects using `setuptools`, this is accomplished by passing `--py-limited-api=cp3x` (where `x` is the minimum Python version supported by the wheel, e.g. `--py-limited-api=cp35` for Python 3.5) to `setup.py bdist_wheel`.
#### Minimum Python version for `abi3` and `abi3t` builds
Because a single `abi3` wheel can be used with many different Python versions, PyO3 has feature flags `abi3-py38`, `abi3-py39`, `abi3-py310` etc. to set the minimum required Python version for your `abi3` wheel.
For example, if you set the `abi3-py38` feature, your wheel can be used on all Python 3 versions from Python 3.8 and up.
`maturin` and `setuptools-rust` will give the wheel a name like `my-extension-1.0-cp38-abi3-manylinux2020_x86_64.whl`.
Similarly, there is an `abi3t-py315` feature and future PyO3 versions will offer `abi3t-py316` and so on.
As your extension module may be run with multiple different Python versions you may occasionally find you need to check the Python version at runtime to customize behavior.
See [the relevant section of this guide](./building-and-distribution/multiple-python-versions.md#checking-the-python-version-at-runtime) on supporting multiple Python versions at runtime.
PyO3 is only able to link your extension module to abi3 version up to and including your host Python version.
E.g., if you set `abi3-py39` and try to compile the crate with a host of Python 3.8, the build will fail.
> [!NOTE]
> If you set more that one of these `abi3` version feature flags the lowest version always wins.
> For example, with both `abi3-py38` and `abi3-py39` set, PyO3 would build a wheel which supports Python 3.8 and up.
#### Building stable ABI extension modules without a Python interpreter
As an advanced feature, you can build a PyO3 wheel without calling Python interpreter with the environment variable `PYO3_NO_PYTHON` set.
Also, if the build host Python interpreter is not found or is too old or otherwise unusable, PyO3 will still attempt to compile stable ABI extension modules after displaying a warning message.
#### Missing features
Due to limitations in the Python API, there are a few `pyo3` features that do not work when compiling for the stable ABI.
These are:
- `#[pyo3(text_signature = "...")]` does not work on classes until Python 3.10 or greater.
- The `dict` and `weakref` options on classes are not supported until Python 3.9 or greater.
- The buffer API is not supported until Python 3.11 or greater.
- Subclassing native types (e.g. `PyException`) is not supported until Python 3.12 or greater.
- Optimizations which rely on knowledge of the exact Python version compiled against.
## Embedding Python in Rust
If you want to embed the Python interpreter inside a Rust program, there are two modes in which this can be done: dynamically and statically.
We'll cover each of these modes in the following sections.
Each of them affect how you must distribute your program.
Instead of learning how to do this yourself, you might want to consider using a project like [PyOxidizer] to ship your application and all of its dependencies in a single file.
PyO3 automatically switches between the two linking modes depending on whether the Python distribution you have configured PyO3 to use ([see above](#configuring-the-python-version)) contains a shared library or a static library.
The static library is most often seen in Python distributions compiled from source without the `--enable-shared` configuration option.
### Dynamically embedding the Python interpreter
Embedding the Python interpreter dynamically is much easier than doing so statically.
This is done by linking your program against a Python shared library (such as `libpython.3.9.so` on UNIX, or `python39.dll` on Windows).
The implementation of the Python interpreter resides inside the shared library.
This means that when the OS runs your Rust program it also needs to be able to find the Python shared library.
This mode of embedding works well for Rust tests which need access to the Python interpreter.
It is also great for Rust software which is installed inside a Python virtualenv, because the virtualenv sets up appropriate environment variables to locate the correct Python shared library.
For distributing your program to non-technical users, you will have to consider including the Python shared library in your distribution as well as setting up wrapper scripts to set the right environment variables (such as `LD_LIBRARY_PATH` on UNIX, or `PATH` on Windows).
Note that PyPy cannot be embedded in Rust (or any other software).
Support for this is tracked on the [PyPy issue tracker](https://github.com/pypy/pypy/issues/3836).
### Statically embedding the Python interpreter
Embedding the Python interpreter statically means including the contents of a Python static library directly inside your Rust binary.
This means that to distribute your program you only need to ship your binary file: it contains the Python interpreter inside the binary!
On Windows static linking is almost never done, so Python distributions don't usually include a static library.
The information below applies only to UNIX.
The Python static library is usually called `libpython.a`.
Static linking has a lot of complications, listed below.
For these reasons PyO3 does not yet have first-class support for this embedding mode.
See [issue 416 on PyO3's GitHub](https://github.com/PyO3/pyo3/issues/416) for more information and to discuss any issues you encounter.
The [`auto-initialize`](features.md#auto-initialize) feature is deliberately disabled when embedding the interpreter statically because this is often unintentionally done by new users to PyO3 running test programs.
Trying out PyO3 is much easier using dynamic embedding.
The known complications are:
- To import compiled extension modules (such as other Rust extension modules, or those written in C), your binary must have the correct linker flags set during compilation to export the original contents of `libpython.a` so that extensions can use them (e.g. `-Wl,--export-dynamic`).
- The C compiler and flags which were used to create `libpython.a` must be compatible with your Rust compiler and flags, else you will experience compilation failures.
Significantly different compiler versions may see errors like this:
```text
lto1: fatal error: bytecode stream in file 'rust-numpy/target/release/deps/libpyo3-6a7fb2ed970dbf26.rlib' generated with LTO version 6.0 instead of the expected 6.2
```
Mismatching flags may lead to errors like this:
```text
/usr/bin/ld: /usr/lib/gcc/x86_64-linux-gnu/9/../../../x86_64-linux-gnu/libpython3.9.a(zlibmodule.o): relocation R_X86_64_32 against `.data' can not be used when making a PIE object; recompile with -fPIE
```
If you encounter these or other complications when linking the interpreter statically, discuss them on [issue 416 on PyO3's GitHub](https://github.com/PyO3/pyo3/issues/416).
It is hoped that eventually that discussion will contain enough information and solutions that PyO3 can offer first-class support for static embedding.
### Import your module when embedding the Python interpreter
When you run your Rust binary with an embedded interpreter, any `#[pymodule]` created modules won't be accessible to import unless added to a table called `PyImport_Inittab` before the embedded interpreter is initialized.
This will cause Python statements in your embedded interpreter such as `import your_new_module` to fail.
You can call the macro [`append_to_inittab`]({{#PYO3_DOCS_URL}}/pyo3/macro.append_to_inittab.html) with your module before initializing the Python interpreter to add the module function into that table. (The Python interpreter will be initialized by calling `Python::initialize`, `with_embedded_python_interpreter`, or `Python::attach` with the [`auto-initialize`](features.md#auto-initialize) feature enabled.)
## Cross Compiling
Thanks to Rust's great cross-compilation support, cross-compiling using PyO3 is relatively straightforward.
To get started, you'll need a few pieces of software:
- A toolchain for your target.
- The appropriate options in your Cargo `.config` for the platform you're targeting and the toolchain you are using.
- A Python interpreter that's already been compiled for your target (optional when building "abi3" extension modules).
- A Python interpreter that is built for your host and available through the `PATH` or setting the [`PYO3_PYTHON`](#configuring-the-python-version) variable (optional when building "abi3" extension modules).
After you've obtained the above, you can build a cross-compiled PyO3 module by using Cargo's `--target` flag.
PyO3's build script will detect that you are attempting a cross-compile based on your host machine and the desired target.
When cross-compiling, PyO3's build script cannot execute the target Python interpreter to query the configuration, so there are a few additional environment variables you may need to set:
- `PYO3_CROSS`: If present this variable forces PyO3 to configure as a cross-compilation.
- `PYO3_CROSS_LIB_DIR`: This variable can be set to the directory containing the target's libpython DSO and the associated `_sysconfigdata*.py` file for Unix-like targets.
This variable is only needed when the output binary must link to libpython explicitly (e.g. when targeting Android, Windows when `raw-dylib` linking is unavailable, or embedding a Python interpreter), or when it is absolutely required to get the interpreter configuration from `_sysconfigdata*.py`.
- `PYO3_CROSS_PYTHON_VERSION`: Major and minor version (e.g. 3.9) of the target Python installation.
This variable is only needed if PyO3 cannot determine the version to target from `abi3-py3*` features, or if `PYO3_CROSS_LIB_DIR` is not set, or if there are multiple versions of Python present in `PYO3_CROSS_LIB_DIR`.
- `PYO3_CROSS_PYTHON_IMPLEMENTATION`: Python implementation name ("CPython" or "PyPy") of the target Python installation.
CPython is assumed by default when this variable is not set, unless `PYO3_CROSS_LIB_DIR` is set for a Unix-like target and PyO3 can get the interpreter configuration from `_sysconfigdata*.py`.
An example might look like the following (assuming your target's sysroot is at `/home/pyo3/cross/sysroot` and that your target is `armv7`):
```sh
export PYO3_CROSS_LIB_DIR="/home/pyo3/cross/sysroot/usr/lib"
cargo build --target armv7-unknown-linux-gnueabihf
```
If there are multiple python versions at the cross lib directory and you cannot set a more precise location to include both the `libpython` DSO and `_sysconfigdata*.py` files, you can set the required version:
```sh
export PYO3_CROSS_PYTHON_VERSION=3.8
export PYO3_CROSS_LIB_DIR="/home/pyo3/cross/sysroot/usr/lib"
cargo build --target armv7-unknown-linux-gnueabihf
```
Any of the `abi3-py3*` features can be enabled instead of setting `PYO3_CROSS_PYTHON_VERSION` in the above examples.
`PYO3_CROSS_LIB_DIR` can often be omitted when cross compiling extension modules for Unix, macOS, and Windows targets.
The following resources may also be useful for cross-compiling:
- [github.com/japaric/rust-cross](https://github.com/japaric/rust-cross) is a primer on cross compiling Rust.
- [github.com/rust-embedded/cross](https://github.com/rust-embedded/cross) uses Docker to make Rust cross-compilation easier.
[`pyo3-build-config`]: https://github.com/PyO3/pyo3/tree/main/pyo3-build-config
[`maturin-starter`]: https://github.com/PyO3/pyo3/tree/main/examples/maturin-starter
[`setuptools-rust-starter`]: https://github.com/PyO3/pyo3/tree/main/examples/setuptools-rust-starter
[`maturin`]: https://github.com/PyO3/maturin
[`setuptools-rust`]: https://github.com/PyO3/setuptools-rust
[PyOxidizer]: https://github.com/indygreg/PyOxidizer
@@ -0,0 +1,133 @@
# Supporting multiple Python versions
PyO3 supports all actively-supported Python 3 and PyPy versions.
As much as possible, this is done internally to PyO3 so that your crate's code does not need to adapt to the differences between each version.
However, as Python features grow and change between versions, PyO3 cannot offer a completely identical API for every Python version.
This may require you to add conditional compilation to your crate or runtime checks for the Python version.
This section of the guide first introduces the `pyo3-build-config` crate, which you can use as a `build-dependency` to add additional `#[cfg]` flags which allow you to support multiple Python versions at compile-time.
Second, we'll show how to check the Python version at runtime.
This can be useful when building for multiple versions with the `abi3` feature, where the Python API compiled against is not always the same as the one in use.
## Conditional compilation for different Python versions
The `pyo3-build-config` exposes multiple [`#[cfg]` flags](https://doc.rust-lang.org/rust-by-example/attribute/cfg.html) which can be used to conditionally compile code for a given Python version.
PyO3 itself depends on this crate, so by using it you can be sure that you are configured correctly for the Python version PyO3 is building against.
This allows us to write code like the following
```rust,ignore
#[cfg(Py_3_8)]
fn function_only_supported_on_python_3_8_and_up() {}
#[cfg(not(Py_3_8))]
fn function_only_supported_before_python_3_8() {}
#[cfg(not(Py_LIMITED_API))]
fn function_incompatible_with_stable_abi_feature() {}
```
The following sections first show how to add these `#[cfg]` flags to your build process, and then cover some common patterns flags in a little more detail.
To see a full reference of all the `#[cfg]` flags provided, see the [`pyo3-build-cfg` docs](https://docs.rs/pyo3-build-config).
### Using `pyo3-build-config`
You can use the `#[cfg]` flags in just two steps:
1. Add `pyo3-build-config` to your crate's build dependencies in `Cargo.toml`:
```toml
[build-dependencies]
pyo3-build-config = {{#PYO3_CRATE_VERSION}}
```
2. Add a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) file to your crate with the following contents:
```rust,ignore
fn main() {
// If you have an existing build.rs file, just add this line to it.
pyo3_build_config::use_pyo3_cfgs();
}
```
After these steps you are ready to annotate your code!
### Common usages of `pyo3-build-cfg` flags
The `#[cfg]` flags added by `pyo3-build-cfg` can be combined with all of Rust's logic in the `#[cfg]` attribute to create very precise conditional code generation.
The following are some common patterns implemented using these flags:
```text
#[cfg(Py_3_8)]
```
This `#[cfg]` marks code that will only be present on Python 3.8 and upwards.
There are similar options `Py_3_9`, `Py_3_10`, `Py_3_11` and so on for each minor version.
```text
#[cfg(not(Py_3_8))]
```
This `#[cfg]` marks code that will only be present on Python versions before (but not including) Python 3.8.
```text
#[cfg(Py_GIL_DISABLED)]
```
This `#[cfg]` marks code that will only be present for free-threaded builds of CPython.
You might use this in situations where the GIL provides thread safety but the free-threaded build needs extra synchronization logic.
```text
#[cfg(not(Py_LIMITED_API))]
```
This `#[cfg]` marks code that is only available when building for the unlimited Python API (i.e. PyO3's `abi3` or `abi3t` features are not enabled).
This might be useful if you want to ship your extension module as an `abi3` or `abi3t` wheel and also allow users to compile it from source to make use of optimizations only possible with the unlimited API.
```text
#[cfg(any(Py_3_9, not(Py_LIMITED_API)))]
```
This `#[cfg]` marks code which is available when running Python 3.9 or newer, or when using the unlimited API with an older Python version.
Patterns like this are commonly seen on Python APIs which were added to the limited Python API in a specific minor version.
```text
#[cfg(all(Py_GIL_DISABLED, Py_LIMITED_API))]
```
This marks code which is available only for the free-threaded `abi3t` stable ABI.
You might use this if you need some sort of custom locking implementation in code that makes strong assumptions about the GIL providing thread safety.
```text
#[cfg(PyPy)]
```
This `#[cfg]` marks code which is running on PyPy.
## Checking the Python version at runtime
When building with PyO3's `abi3` or `abi3t` features, your extension module will be compiled against a specific [minimum version](../building-and-distribution.md#minimum-python-version-for-abi3-and-abi3t-builds) of Python, but may be running on newer Python versions.
For example with PyO3's `abi3-py38` feature, your extension module will be compiled as if it were for Python 3.8.
If you were using `pyo3-build-config`, `#[cfg(Py_3_8)]` would be present.
Your user could freely install and run your abi3 extension module on Python 3.9.
There's no way to detect your user doing that at compile time, so instead you need to fall back to runtime checks.
PyO3 provides the APIs [`Python::version()`] and [`Python::version_info()`] to query the running Python version.
This allows you to do the following, for example:
```rust
use pyo3::Python;
Python::attach(|py| {
// PyO3 supports Python 3.8 and up.
assert!(py.version_info() >= (3, 8));
assert!(py.version_info() >= (3, 8, 0));
});
```
[`Python::version()`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.version
[`Python::version_info()`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.version_info
+1
View File
@@ -0,0 +1 @@
{{#include ../../CHANGELOG.md}}
+1589
View File
File diff suppressed because it is too large. Load diff
+126
View File
@@ -0,0 +1,126 @@
# Emulating callable objects
Classes can be callable if they have a `#[pymethod]` named `__call__`.
This allows instances of a class to behave similar to functions.
This method's signature must look like `__call__(<self>, ...) -> object` - here, any argument list can be defined as for normal pymethods
## Example: Implementing a call counter
The following pyclass is a basic decorator - its constructor takes a Python object as argument and calls that object when called.
An equivalent Python implementation is linked at the end.
An example crate containing this pyclass can be found [in the PyO3 GitHub repository](https://github.com/PyO3/pyo3/tree/main/examples/decorator)
```rust,ignore
{{#include ../../../examples/decorator/src/lib.rs}}
```
Python code:
```python
{{#include ../../../examples/decorator/tests/example.py}}
```
Output:
```text
say_hello has been called 1 time(s).
hello
say_hello has been called 2 time(s).
hello
say_hello has been called 3 time(s).
hello
say_hello has been called 4 time(s).
hello
```
### Pure Python implementation
A Python implementation of this looks similar to the Rust version:
```python
class Counter:
def __init__(self, wraps):
self.count = 0
self.wraps = wraps
def __call__(self, *args, **kwargs):
self.count += 1
print(f"{self.wraps.__name__} has been called {self.count} time(s)")
self.wraps(*args, **kwargs)
```
Note that it can also be implemented as a higher order function:
```python
def Counter(wraps):
count = 0
def call(*args, **kwargs):
nonlocal count
count += 1
print(f"{wraps.__name__} has been called {count} time(s)")
return wraps(*args, **kwargs)
return call
```
### What is the `AtomicU64` for?
A [previous implementation] used a normal `u64`, which meant it required a `&mut self` receiver to update the count:
```rust,ignore
#[pyo3(signature = (*args, **kwargs))]
fn __call__(
&mut self,
py: Python<'_>,
args: &Bound<'_, PyTuple>,
kwargs: Option<&Bound<'_, PyDict>>,
) -> PyResult<Py<PyAny>> {
self.count += 1;
let name = self.wraps.getattr(py, "__name__")?;
println!("{} has been called {} time(s).", name, self.count);
// After doing something, we finally forward the call to the wrapped function
let ret = self.wraps.call(py, args, kwargs)?;
// We could do something with the return value of
// the function before returning it
Ok(ret)
}
```
The problem with this is that the `&mut self` receiver means PyO3 has to borrow it exclusively, and hold this borrow across the `self.wraps.call(py, args, kwargs)` call.
This call returns control to the user's Python code which is free to call arbitrary things, *including* the decorated function.
If that happens PyO3 is unable to create a second unique borrow and will be forced to raise an exception.
As a result, something innocent like this will raise an exception:
```py
@Counter
def say_hello():
if say_hello.count < 2:
print(f"hello from decorator")
say_hello()
# RuntimeError: Already borrowed
```
The implementation in this chapter fixes that by never borrowing exclusively; all the methods take `&self` as receivers, of which multiple may exist simultaneously.
This requires a shared counter and the most straightforward way to implement thread-safe interior mutability (e.g. the type does not need to accept `&mut self` to modify the "interior" state) for a `u64` is to use [`AtomicU64`], so that's what is used here.
This shows the dangers of running arbitrary Python code - note that "running arbitrary Python code" can be far more subtle than the example above:
- Python's asynchronous executor may park the current thread in the middle of Python code, even in Python code that *you* control, and let other Python code run.
- Dropping arbitrary Python objects may invoke destructors defined in Python (`__del__` methods).
- Calling Python's C-api (most PyO3 apis call C-api functions internally) may raise exceptions, which may allow Python code in signal handlers to run.
- On the free-threaded build, users might use Python's `threading` module to work with your types simultaneously from multiple OS threads.
This is especially important if you are writing unsafe code; Python code must never be able to cause undefined behavior.
You must ensure that your Rust code is in a consistent state before doing any of the above things.
<!-- rumdl-disable MD057 -->
<!-- TODO: investigate why this lint is being triggered -->
[previous implementation]: <https://github.com/PyO3/pyo3/discussions/2598> "Thread Safe Decorator <Help Wanted> · Discussion #2598 · PyO3/pyo3"
[`AtomicU64`]: <https://doc.rust-lang.org/std/sync/atomic/type.AtomicU64.html> "AtomicU64 in std::sync::atomic - Rust"
<!-- rumdl-enable MD057 -->
+451
View File
@@ -0,0 +1,451 @@
# Emulating numeric types
At this point we have a `Number` class that we can't actually do any math on!
Before proceeding, we should think about how we want to handle overflows.
There are three obvious solutions:
- We can have infinite precision just like Python's `int`.
<!-- no-glossary-link:wheel -->
However that would be quite boring - we'd be reinventing the wheel.
- We can raise exceptions whenever `Number` overflows, but that makes the API painful to use.
- We can wrap around the boundary of `i32`.
This is the approach we'll take here.
To do that we'll just forward to `i32`'s `wrapping_*` methods.
## Fixing our constructor
Let's address the first overflow, in `Number`'s constructor:
```python
from my_module import Number
n = Number(1 << 1337)
```
```text
Traceback (most recent call last):
File "example.py", line 3, in <module>
n = Number(1 << 1337)
OverflowError: Python int too large to convert to C long
```
Instead of relying on the default [`FromPyObject`] extraction to parse arguments, we can specify our own extraction function, using the `#[pyo3(from_py_with = ...)]` attribute.
Unfortunately PyO3 doesn't provide a way to wrap Python integers out of the box, but we can do a Python call to mask it and cast it to an `i32`.
```rust,no_run
# #![allow(dead_code)]
use pyo3::prelude::*;
fn wrap(obj: &Bound<'_, PyAny>) -> PyResult<i32> {
let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?;
let val: u32 = val.extract()?;
// 👇 This intentionally overflows!
Ok(val as i32)
}
```
We also add documentation, via `///` comments, which are visible to Python users.
```rust,no_run
# #![allow(dead_code)]
use pyo3::prelude::*;
fn wrap(obj: &Bound<'_, PyAny>) -> PyResult<i32> {
let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?;
let val: u32 = val.extract()?;
Ok(val as i32)
}
/// Did you ever hear the tragedy of Darth Signed The Overfloweth? I thought not.
/// It's not a story C would tell you. It's a Rust legend.
#[pyclass(module = "my_module")]
struct Number(i32);
#[pymethods]
impl Number {
#[new]
fn new(#[pyo3(from_py_with = wrap)] value: i32) -> Self {
Self(value)
}
}
```
With that out of the way, let's implement some operators:
```rust,no_run
use pyo3::exceptions::{PyZeroDivisionError, PyValueError};
# use pyo3::prelude::*;
#
# #[pyclass]
# struct Number(i32);
#
#[pymethods]
impl Number {
fn __add__(&self, other: &Self) -> Self {
Self(self.0.wrapping_add(other.0))
}
fn __sub__(&self, other: &Self) -> Self {
Self(self.0.wrapping_sub(other.0))
}
fn __mul__(&self, other: &Self) -> Self {
Self(self.0.wrapping_mul(other.0))
}
fn __truediv__(&self, other: &Self) -> PyResult<Self> {
match self.0.checked_div(other.0) {
Some(i) => Ok(Self(i)),
None => Err(PyZeroDivisionError::new_err("division by zero")),
}
}
fn __floordiv__(&self, other: &Self) -> PyResult<Self> {
match self.0.checked_div(other.0) {
Some(i) => Ok(Self(i)),
None => Err(PyZeroDivisionError::new_err("division by zero")),
}
}
fn __rshift__(&self, other: &Self) -> PyResult<Self> {
match other.0.try_into() {
Ok(rhs) => Ok(Self(self.0.wrapping_shr(rhs))),
Err(_) => Err(PyValueError::new_err("negative shift count")),
}
}
fn __lshift__(&self, other: &Self) -> PyResult<Self> {
match other.0.try_into() {
Ok(rhs) => Ok(Self(self.0.wrapping_shl(rhs))),
Err(_) => Err(PyValueError::new_err("negative shift count")),
}
}
}
```
### Unary arithmetic operations
```rust,no_run
# use pyo3::prelude::*;
#
# #[pyclass]
# struct Number(i32);
#
#[pymethods]
impl Number {
fn __pos__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
slf
}
fn __neg__(&self) -> Self {
Self(-self.0)
}
fn __abs__(&self) -> Self {
Self(self.0.abs())
}
fn __invert__(&self) -> Self {
Self(!self.0)
}
}
```
### Support for the `complex()`, `int()` and `float()` built-in functions
```rust,no_run
# use pyo3::prelude::*;
#
# #[pyclass]
# struct Number(i32);
#
use pyo3::types::PyComplex;
#[pymethods]
impl Number {
fn __int__(&self) -> i32 {
self.0
}
fn __float__(&self) -> f64 {
self.0 as f64
}
fn __complex__<'py>(&self, py: Python<'py>) -> Bound<'py, PyComplex> {
PyComplex::from_doubles(py, self.0 as f64, 0.0)
}
}
```
We do not implement the in-place operations like `__iadd__` because we do not wish to mutate `Number`.
Similarly we're not interested in supporting operations with different types, so we do not implement the reflected operations like `__radd__` either.
Now Python can use our `Number` class:
```python
from my_module import Number
def hash_djb2(s: str):
'''
A version of Daniel J. Bernstein's djb2 string hashing algorithm
Like many hashing algorithms, it relies on integer wrapping.
'''
n = Number(0)
five = Number(5)
for x in s:
n = Number(ord(x)) + ((n << five) - n)
return n
assert hash_djb2('l50_50') == Number(-1152549421)
```
### Final code
```rust
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use pyo3::exceptions::{PyValueError, PyZeroDivisionError};
use pyo3::prelude::*;
use pyo3::class::basic::CompareOp;
use pyo3::types::{PyComplex, PyString};
fn wrap(obj: &Bound<'_, PyAny>) -> PyResult<i32> {
let val = obj.call_method1("__and__", (0xFFFFFFFF_u32,))?;
let val: u32 = val.extract()?;
Ok(val as i32)
}
/// Did you ever hear the tragedy of Darth Signed The Overfloweth? I thought not.
/// It's not a story C would tell you. It's a Rust legend.
#[pyclass(module = "my_module")]
struct Number(i32);
#[pymethods]
impl Number {
#[new]
fn new(#[pyo3(from_py_with = wrap)] value: i32) -> Self {
Self(value)
}
fn __repr__(slf: &Bound<'_, Self>) -> PyResult<String> {
// Get the class name dynamically in case `Number` is subclassed
let class_name: Bound<'_, PyString> = slf.get_type().qualname()?;
Ok(format!("{}({})", class_name, slf.borrow().0))
}
fn __str__(&self) -> String {
self.0.to_string()
}
fn __hash__(&self) -> u64 {
let mut hasher = DefaultHasher::new();
self.0.hash(&mut hasher);
hasher.finish()
}
fn __richcmp__(&self, other: &Self, op: CompareOp) -> PyResult<bool> {
match op {
CompareOp::Lt => Ok(self.0 < other.0),
CompareOp::Le => Ok(self.0 <= other.0),
CompareOp::Eq => Ok(self.0 == other.0),
CompareOp::Ne => Ok(self.0 != other.0),
CompareOp::Gt => Ok(self.0 > other.0),
CompareOp::Ge => Ok(self.0 >= other.0),
}
}
fn __bool__(&self) -> bool {
self.0 != 0
}
fn __add__(&self, other: &Self) -> Self {
Self(self.0.wrapping_add(other.0))
}
fn __sub__(&self, other: &Self) -> Self {
Self(self.0.wrapping_sub(other.0))
}
fn __mul__(&self, other: &Self) -> Self {
Self(self.0.wrapping_mul(other.0))
}
fn __truediv__(&self, other: &Self) -> PyResult<Self> {
match self.0.checked_div(other.0) {
Some(i) => Ok(Self(i)),
None => Err(PyZeroDivisionError::new_err("division by zero")),
}
}
fn __floordiv__(&self, other: &Self) -> PyResult<Self> {
match self.0.checked_div(other.0) {
Some(i) => Ok(Self(i)),
None => Err(PyZeroDivisionError::new_err("division by zero")),
}
}
fn __rshift__(&self, other: &Self) -> PyResult<Self> {
match other.0.try_into() {
Ok(rhs) => Ok(Self(self.0.wrapping_shr(rhs))),
Err(_) => Err(PyValueError::new_err("negative shift count")),
}
}
fn __lshift__(&self, other: &Self) -> PyResult<Self> {
match other.0.try_into() {
Ok(rhs) => Ok(Self(self.0.wrapping_shl(rhs))),
Err(_) => Err(PyValueError::new_err("negative shift count")),
}
}
fn __xor__(&self, other: &Self) -> Self {
Self(self.0 ^ other.0)
}
fn __or__(&self, other: &Self) -> Self {
Self(self.0 | other.0)
}
fn __and__(&self, other: &Self) -> Self {
Self(self.0 & other.0)
}
fn __int__(&self) -> i32 {
self.0
}
fn __float__(&self) -> f64 {
self.0 as f64
}
fn __complex__<'py>(&self, py: Python<'py>) -> Bound<'py, PyComplex> {
PyComplex::from_doubles(py, self.0 as f64, 0.0)
}
}
#[pymodule]
mod my_module {
#[pymodule_export]
use super::Number;
}
# const SCRIPT: &'static std::ffi::CStr = cr#"
# def hash_djb2(s: str):
# n = Number(0)
# five = Number(5)
#
# for x in s:
# n = Number(ord(x)) + ((n << five) - n)
# return n
#
# assert hash_djb2('l50_50') == Number(-1152549421)
# assert hash_djb2('logo') == Number(3327403)
# assert hash_djb2('horizon') == Number(1097468315)
#
#
# assert Number(2) + Number(2) == Number(4)
# assert Number(2) + Number(2) != Number(5)
#
# assert Number(13) - Number(7) == Number(6)
# assert Number(13) - Number(-7) == Number(20)
#
# assert Number(13) / Number(7) == Number(1)
# assert Number(13) // Number(7) == Number(1)
#
# assert Number(13) * Number(7) == Number(13*7)
#
# assert Number(13) > Number(7)
# assert Number(13) < Number(20)
# assert Number(13) == Number(13)
# assert Number(13) >= Number(7)
# assert Number(13) <= Number(20)
# assert Number(13) == Number(13)
#
#
# assert (True if Number(1) else False)
# assert (False if Number(0) else True)
#
#
# assert int(Number(13)) == 13
# assert float(Number(13)) == 13
# assert Number.__doc__ == "Did you ever hear the tragedy of Darth Signed The Overfloweth? I thought not.\nIt's not a story C would tell you. It's a Rust legend."
# assert Number(12345234523452) == Number(1498514748)
# try:
# import inspect
# assert inspect.signature(Number).__str__() == '(value)'
# except ValueError:
# # Not supported with `abi3` before Python 3.10
# pass
# assert Number(1337).__str__() == '1337'
# assert Number(1337).__repr__() == 'Number(1337)'
"#;
#
# use pyo3::PyTypeInfo;
#
# fn main() -> PyResult<()> {
# Python::attach(|py| -> PyResult<()> {
# let globals = PyModule::import(py, "__main__")?.dict();
# globals.set_item("Number", Number::type_object(py))?;
#
# py.run(SCRIPT, Some(&globals), None)?;
# Ok(())
# })
# }
```
## Appendix: Writing some unsafe code
At the beginning of this chapter we said that PyO3 doesn't provide a way to wrap Python integers out of the box but that's a half truth.
There's not a PyO3 API for it, but there's a Python C API function that does:
```c
unsigned long PyLong_AsUnsignedLongMask(PyObject *obj)
```
We can call this function from Rust by using [`pyo3::ffi::PyLong_AsUnsignedLongMask`].
This is an *unsafe* function, which means we have to use an unsafe block to call it and take responsibility for upholding the contracts of this function.
Let's review those contracts:
- We must be attached to the interpreter.
If we're not, calling this function causes a data race.
- The pointer must be valid, i.e. it must be properly aligned and point to a valid Python object.
Let's create that helper function.
The signature has to be `fn(&Bound<'_, PyAny>) -> PyResult<T>`.
- `&Bound<'_, PyAny>` represents a checked bound reference, so the pointer derived from it is valid (and not null).
- Whenever we have bound references to Python objects in scope, it is guaranteed that we're attached to the interpreter.
This reference is also where we can get a [`Python`] token to use in our call to [`PyErr::take`].
```rust,no_run
# #![allow(dead_code)]
use std::ffi::c_ulong;
use pyo3::prelude::*;
use pyo3::ffi;
fn wrap(obj: &Bound<'_, PyAny>) -> Result<i32, PyErr> {
let py: Python<'_> = obj.py();
unsafe {
let ptr = obj.as_ptr();
let ret: c_ulong = ffi::PyLong_AsUnsignedLongMask(ptr);
if ret == c_ulong::MAX {
if let Some(err) = PyErr::take(py) {
return Err(err);
}
}
Ok(ret as i32)
}
}
```
[`PyErr::take`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/struct.PyErr.html#method.take
[`Python`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html
[`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html
[`pyo3::ffi::PyLong_AsUnsignedLongMask`]: {{#PYO3_DOCS_URL}}/pyo3/ffi/fn.PyLong_AsUnsignedLongMask.html
+394
View File
@@ -0,0 +1,394 @@
# Basic object customization
Recall the `Number` class from the previous chapter:
```rust,no_run
# #![allow(dead_code)]
# fn main() {}
use pyo3::prelude::*;
#[pyclass]
struct Number(i32);
#[pymethods]
impl Number {
#[new]
fn new(value: i32) -> Self {
Self(value)
}
}
#[pymodule]
mod my_module {
#[pymodule_export]
use super::Number;
}
```
At this point Python code can import the module, access the class and create class instances - but nothing else.
```python
from my_module import Number
n = Number(5)
print(n)
```
```text
<builtins.Number object at 0x000002B4D185D7D0>
```
## String representations
It can't even print an user-readable representation of itself!
We can fix that by defining the `__repr__` and `__str__` methods inside a `#[pymethods]` block.
We do this by accessing the value contained inside `Number`.
```rust,no_run
# use pyo3::prelude::*;
#
# #[pyclass]
# struct Number(i32);
#
#[pymethods]
impl Number {
// For `__repr__` we want to return a string that Python code could use to recreate
// the `Number`, like `Number(5)` for example.
fn __repr__(&self) -> String {
// We use the `format!` macro to create a string. Its first argument is a
// format string, followed by any number of parameters which replace the
// `{}`'s in the format string.
//
// 👇 Tuple field access in Rust uses a dot
format!("Number({})", self.0)
}
// `__str__` is generally used to create an "informal" representation, so we
// just forward to `i32`'s `ToString` trait implementation to print a bare number.
fn __str__(&self) -> String {
self.0.to_string()
}
}
```
To automatically generate the `__str__` implementation using a `Display` trait implementation, pass the `str` argument to `pyclass`.
```rust,no_run
# use std::fmt::{Display, Formatter};
# use pyo3::prelude::*;
#
# #[allow(dead_code)]
#[pyclass(str)]
struct Coordinate {
x: i32,
y: i32,
z: i32,
}
impl Display for Coordinate {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "({}, {}, {})", self.x, self.y, self.z)
}
}
```
For convenience, a shorthand format string can be passed to `str` as `str="<format string>"` for **structs only**.
It expands and is passed into the `format!` macro in the following ways:
- `"{x}"` -> `"{}", self.x`
- `"{0}"` -> `"{}", self.0`
- `"{x:?}"` -> `"{:?}", self.x`
*Note: Depending upon the format string you use, this may require implementation of the `Display` or `Debug` traits for the given Rust types.*
*Note: the pyclass args `name` and `rename_all` are incompatible with the shorthand format string and will raise a compile time error.*
```rust,no_run
# use pyo3::prelude::*;
#
# #[allow(dead_code)]
#[pyclass(str="({x}, {y}, {z})")]
struct Coordinate {
x: i32,
y: i32,
z: i32,
}
```
### Accessing the class name
In the `__repr__`, we used a hard-coded class name.
This is sometimes not ideal, because if the class is subclassed in Python, we would like the repr to reflect the subclass name.
This is typically done in Python code by accessing `self.__class__.__name__`.
In order to be able to access the Python type information *and* the Rust struct, we need to use a `Bound` as the `self` argument.
```rust,no_run
# use pyo3::prelude::*;
# use pyo3::types::PyString;
#
# #[allow(dead_code)]
# #[pyclass]
# struct Number(i32);
#
#[pymethods]
impl Number {
fn __repr__(slf: &Bound<'_, Self>) -> PyResult<String> {
// This is the equivalent of `self.__class__.__name__` in Python.
let class_name: Bound<'_, PyString> = slf.get_type().qualname()?;
// To access fields of the Rust struct, we need to borrow from the Bound object.
Ok(format!("{}({})", class_name, slf.borrow().0))
}
}
```
### Hashing
Let's also implement hashing.
We'll just hash the `i32`.
For that we need a [`Hasher`].
The one provided by `std` is [`DefaultHasher`], which uses the [SipHash] algorithm.
```rust,no_run
use std::collections::hash_map::DefaultHasher;
// Required to call the `.hash` and `.finish` methods, which are defined on traits.
use std::hash::{Hash, Hasher};
# use pyo3::prelude::*;
#
# #[allow(dead_code)]
# #[pyclass]
# struct Number(i32);
#
#[pymethods]
impl Number {
fn __hash__(&self) -> u64 {
let mut hasher = DefaultHasher::new();
self.0.hash(&mut hasher);
hasher.finish()
}
}
```
To implement `__hash__` using the Rust [`Hash`] trait implementation, the `hash` option can be used.
This option is only available for `frozen` classes to prevent accidental hash changes from mutating the object.
If you need an `__hash__` implementation for a mutable class, use the manual method from above.
This option also requires `eq`: According to the [Python docs](https://docs.python.org/3/reference/datamodel.html#object.__hash__) "If a class does not define an `__eq__()` method it should not define a `__hash__()` operation either"
```rust,no_run
# use pyo3::prelude::*;
#
# #[allow(dead_code)]
#[pyclass(frozen, eq, hash)]
#[derive(PartialEq, Hash)]
struct Number(i32);
```
> [!NOTE]
> When implementing `__hash__` and comparisons, it is important that the following property holds:
>
> ```text
> k1 == k2 -> hash(k1) == hash(k2)
> ```
>
> In other words, if two keys are equal, their hashes must also be equal. In addition you must take
> care that your classes' hash doesn't change during its lifetime. In this tutorial we do that by not
> letting Python code change our `Number` class. In other words, it is immutable.
>
> By default, all `#[pyclass]` types have a default hash implementation from Python.
> Types which should not be hashable can override this by setting `__hash__` to None.
> This is the same mechanism as for a pure-Python class. This is done like so:
>
> ```rust,no_run
> # use pyo3::prelude::*;
> #[pyclass]
> struct NotHashable {}
>
> #[pymethods]
> impl NotHashable {
> #[classattr]
> const __hash__: Option<Py<PyAny>> = None;
> }
> ```
### Comparisons
PyO3 supports the usual magic comparison methods available in Python such as `__eq__`, `__lt__` and so on.
It is also possible to support all six operations at once with `__richcmp__`.
This method will be called with a value of `CompareOp` depending on the operation.
```rust,no_run
use pyo3::class::basic::CompareOp;
# use pyo3::prelude::*;
#
# #[allow(dead_code)]
# #[pyclass]
# struct Number(i32);
#
#[pymethods]
impl Number {
fn __richcmp__(&self, other: &Self, op: CompareOp) -> PyResult<bool> {
match op {
CompareOp::Lt => Ok(self.0 < other.0),
CompareOp::Le => Ok(self.0 <= other.0),
CompareOp::Eq => Ok(self.0 == other.0),
CompareOp::Ne => Ok(self.0 != other.0),
CompareOp::Gt => Ok(self.0 > other.0),
CompareOp::Ge => Ok(self.0 >= other.0),
}
}
}
```
If you obtain the result by comparing two Rust values, as in this example, you can take a shortcut using `CompareOp::matches`:
```rust,no_run
use pyo3::class::basic::CompareOp;
# use pyo3::prelude::*;
#
# #[allow(dead_code)]
# #[pyclass]
# struct Number(i32);
#
#[pymethods]
impl Number {
fn __richcmp__(&self, other: &Self, op: CompareOp) -> bool {
op.matches(self.0.cmp(&other.0))
}
}
```
It checks that the `std::cmp::Ordering` obtained from Rust's `Ord` matches the given `CompareOp`.
Alternatively, you can implement just equality using `__eq__`:
```rust
# use pyo3::prelude::*;
#
# #[pyclass]
# struct Number(i32);
#
#[pymethods]
impl Number {
fn __eq__(&self, other: &Self) -> bool {
self.0 == other.0
}
}
# fn main() -> PyResult<()> {
# Python::attach(|py| {
# let x = &Bound::new(py, Number(4))?;
# let y = &Bound::new(py, Number(4))?;
# assert!(x.eq(y)?);
# assert!(!x.ne(y)?);
# Ok(())
# })
# }
```
To implement `__eq__` using the Rust [`PartialEq`] trait implementation, the `eq` option can be used.
```rust,no_run
# use pyo3::prelude::*;
#
# #[allow(dead_code)]
#[pyclass(eq)]
#[derive(PartialEq)]
struct Number(i32);
```
To implement `__lt__`, `__le__`, `__gt__`, & `__ge__` using the Rust `PartialOrd` trait implementation, the `ord` option can be used.
*Note: Requires `eq`.*
```rust,no_run
# use pyo3::prelude::*;
#
# #[allow(dead_code)]
#[pyclass(eq, ord)]
#[derive(PartialEq, PartialOrd)]
struct Number(i32);
```
### Truthyness
We'll consider `Number` to be `True` if it is nonzero:
```rust,no_run
# use pyo3::prelude::*;
#
# #[allow(dead_code)]
# #[pyclass]
# struct Number(i32);
#
#[pymethods]
impl Number {
fn __bool__(&self) -> bool {
self.0 != 0
}
}
```
### Final code
```rust,no_run
# fn main() {}
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
use pyo3::prelude::*;
use pyo3::class::basic::CompareOp;
use pyo3::types::PyString;
#[pyclass]
struct Number(i32);
#[pymethods]
impl Number {
#[new]
fn new(value: i32) -> Self {
Self(value)
}
fn __repr__(slf: &Bound<'_, Self>) -> PyResult<String> {
let class_name: Bound<'_, PyString> = slf.get_type().qualname()?;
Ok(format!("{}({})", class_name, slf.borrow().0))
}
fn __str__(&self) -> String {
self.0.to_string()
}
fn __hash__(&self) -> u64 {
let mut hasher = DefaultHasher::new();
self.0.hash(&mut hasher);
hasher.finish()
}
fn __richcmp__(&self, other: &Self, op: CompareOp) -> PyResult<bool> {
match op {
CompareOp::Lt => Ok(self.0 < other.0),
CompareOp::Le => Ok(self.0 <= other.0),
CompareOp::Eq => Ok(self.0 == other.0),
CompareOp::Ne => Ok(self.0 != other.0),
CompareOp::Gt => Ok(self.0 > other.0),
CompareOp::Ge => Ok(self.0 >= other.0),
}
}
fn __bool__(&self) -> bool {
self.0 != 0
}
}
#[pymodule]
mod my_module {
#[pymodule_export]
use super::Number;
}
```
[`Hash`]: https://doc.rust-lang.org/std/hash/trait.Hash.html
[`Hasher`]: https://doc.rust-lang.org/std/hash/trait.Hasher.html
[`DefaultHasher`]: https://doc.rust-lang.org/std/collections/hash_map/struct.DefaultHasher.html
[SipHash]: https://en.wikipedia.org/wiki/SipHash
[`PartialEq`]: https://doc.rust-lang.org/stable/std/cmp/trait.PartialEq.html
+502
View File
@@ -0,0 +1,502 @@
# Class customizations
Python's object model defines several protocols for different object behavior, such as the sequence, mapping, and number protocols.
Python classes support these protocols by implementing "magic" methods, such as `__str__` or `__repr__`.
Because of the double-underscores surrounding their name, these are also known as "dunder" methods.
PyO3 makes it possible for every magic method to be implemented in `#[pymethods]` just as they would be done in a regular Python class, with a few notable differences:
- `__new__` is replaced by the [`#[new]` attribute](../class.md#constructor).
- `__del__` is not yet supported, but may be in the future.
- `__buffer__` and `__release_buffer__` are currently not supported and instead PyO3 supports [`__getbuffer__` and `__releasebuffer__`](#buffer-objects) methods (these predate [PEP 688](https://peps.python.org/pep-0688/#python-level-buffer-protocol)), again this may change in the future.
- PyO3 adds [`__traverse__` and `__clear__`](#garbage-collector-integration) methods for controlling garbage collection.
- The Python C-API which PyO3 is implemented upon requires many magic methods to have a specific function signature in C and be placed into special "slots" on the class type object.
This limits the allowed argument and return types for these methods.
They are listed in detail in the section below.
If a magic method is not on the list above (for example `__init_subclass__`), then it should just work in PyO3.
If this is not the case, please file a bug report.
## Magic Methods handled by PyO3
If a function name in `#[pymethods]` is a magic method which is known to need special handling, it will be automatically placed into the correct slot in the Python type object.
The function name is taken from the usual rules for naming `#[pymethods]`: the `#[pyo3(name = "...")]` attribute is used if present, otherwise the Rust function name is used.
The magic methods handled by PyO3 are very similar to the standard Python ones on [this page](https://docs.python.org/3/reference/datamodel.html#special-method-names) - in particular they are the subset which have slots as [defined here](https://docs.python.org/3/c-api/typeobj.html).
When PyO3 handles a magic method, a couple of changes apply compared to other `#[pymethods]`:
- The Rust function signature is restricted to match the magic method.
- The `#[pyo3(signature = (...)]` and `#[pyo3(text_signature = "...")]` attributes are not allowed.
The following sections list all magic methods for which PyO3 implements the necessary special handling.
The given signatures should be interpreted as follows:
- All methods take a receiver as first argument, shown as `<self>`.
It can be `&self`, `&mut self` or a `Bound` reference like `self_: PyRef<'_, Self>` and `self_: PyRefMut<'_, Self>`, as described [in the parent section](../class.md#inheritance).
- An optional `Python<'py>` argument is always allowed as the first argument.
- Return values can be optionally wrapped in `PyResult`.
- `object` means that any type is allowed that can be extracted from a Python
object (if argument) or converted to a Python object (if return value).
- Other types must match what's given, e.g. `pyo3::basic::CompareOp` for
`__richcmp__`'s second argument.
- For the comparison and arithmetic methods, extraction errors are not
propagated as exceptions, but lead to a return of `NotImplemented`.
- For some magic methods, the return values are not restricted by PyO3, but checked by the Python interpreter.
For example, `__str__` needs to return a string object.
This is indicated by `object (Python type)`.
### Basic object customization
- `__str__(<self>) -> object (str)`
- `__repr__(<self>) -> object (str)`
<!-- rumdl-disable MD013 -->
<!-- TODO: report false positive -->
- `__hash__(<self>) -> isize`
Objects that compare equal must have the same hash value.
Any type up to 64 bits may be returned instead of `isize`, PyO3 will convert to an isize automatically (wrapping unsigned types like `u64` and `usize`).
<details>
<summary>Disabling Python's default hash</summary>
By default, all `#[pyclass]` types have a default hash implementation from Python.
Types which should not be hashable can override this by setting `__hash__` to `None`.
This is the same mechanism as for a pure-Python class.
This is done like so:
```rust,no_run
# use pyo3::prelude::*;
#
#[pyclass]
struct NotHashable {}
#[pymethods]
impl NotHashable {
#[classattr]
const __hash__: Option<Py<PyAny>> = None;
}
```
</details>
<!-- rumdl-enable MD013 -->
- `__lt__(<self>, object) -> object`
- `__le__(<self>, object) -> object`
- `__eq__(<self>, object) -> object`
- `__ne__(<self>, object) -> object`
- `__gt__(<self>, object) -> object`
- `__ge__(<self>, object) -> object`
The implementations of Python's "rich comparison" operators `<`, `<=`, `==`, `!=`, `>` and `>=` respectively.
*Note that implementing any of these methods will cause Python not to generate a default `__hash__` implementation, so consider also implementing `__hash__`.*
<details>
<summary>Return type</summary>
The return type will normally be `bool` or `PyResult<bool>`, however any Python object can be returned.
</details>
<!-- rumdl-disable MD013 -->
<!-- TODO: report false positive -->
- `__richcmp__(<self>, object, pyo3::basic::CompareOp) -> object`
Implements Python comparison operations (`==`, `!=`, `<`, `<=`, `>`, and `>=`) in a single method.
The `CompareOp` argument indicates the comparison operation being performed.
You can use [`CompareOp::matches`] to adapt a Rust `std::cmp::Ordering` result to the requested comparison.
*This method cannot be implemented in combination with any of `__lt__`, `__le__`, `__eq__`, `__ne__`, `__gt__`, or `__ge__`.*
*Note that implementing `__richcmp__` will cause Python not to generate a default `__hash__` implementation, so consider implementing `__hash__` when implementing `__richcmp__`.*
<details>
<summary>Return type</summary>
The return type will normally be `PyResult<bool>`, but any Python object can be returned.
If you want to leave some operations unimplemented, you can return `py.NotImplemented()`
for some of the operations:
```rust,no_run
use pyo3::class::basic::CompareOp;
use pyo3::types::PyNotImplemented;
# use pyo3::prelude::*;
# use pyo3::BoundObject;
#
# #[pyclass]
# struct Number(i32);
#
#[pymethods]
impl Number {
fn __richcmp__<'py>(&self, other: &Self, op: CompareOp, py: Python<'py>) -> PyResult<Borrowed<'py, 'py, PyAny>> {
match op {
CompareOp::Eq => Ok((self.0 == other.0).into_pyobject(py)?.into_any()),
CompareOp::Ne => Ok((self.0 != other.0).into_pyobject(py)?.into_any()),
_ => Ok(PyNotImplemented::get(py).into_any()),
}
}
}
```
If the second argument `object` is not of the type specified in the signature, the generated code will automatically `return NotImplemented`.
</details>
<!-- rumdl-enable MD013 -->
- `__getattr__(<self>, object) -> object`
- `__getattribute__(<self>, object) -> object`
<details>
<summary>Differences between <code>__getattr__</code> and <code>__getattribute__</code></summary>
As in Python, `__getattr__` is only called if the attribute is not found
by normal attribute lookup. `__getattribute__`, on the other hand, is
called for *every* attribute access. If it wants to access existing
attributes on `self`, it needs to be very careful not to introduce
infinite recursion, and use `baseclass.__getattribute__()`.
</details>
- `__setattr__(<self>, value: object) -> ()`
- `__delattr__(<self>, object) -> ()`
Overrides attribute access.
- `__bool__(<self>) -> bool`
Determines the "truthyness" of an object.
- `__call__(<self>, ...) -> object` - here, any argument list can be defined
as for normal `pymethods`
### Iterable objects
Iterators can be defined using these methods:
- `__iter__(<self>) -> object`
- `__next__(<self>) -> Option<object> or IterNextOutput` ([see details](#returning-a-value-from-iteration))
Returning `None` from `__next__` indicates that that there are no further items.
Example:
```rust,no_run
use pyo3::prelude::*;
use std::sync::Mutex;
#[pyclass]
struct MyIterator {
iter: Mutex<Box<dyn Iterator<Item = Py<PyAny>> + Send>>,
}
#[pymethods]
impl MyIterator {
fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
slf
}
fn __next__(slf: PyRefMut<'_, Self>) -> Option<Py<PyAny>> {
slf.iter.lock().unwrap().next()
}
}
```
In many cases you'll have a distinction between the type being iterated over (i.e. the *iterable*) and the iterator it provides.
In this case, the iterable only needs to implement `__iter__()` while the iterator must implement both `__iter__()` and `__next__()`.
For example:
```rust,no_run
# use pyo3::prelude::*;
#[pyclass]
struct Iter {
inner: std::vec::IntoIter<usize>,
}
#[pymethods]
impl Iter {
fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> {
slf
}
fn __next__(mut slf: PyRefMut<'_, Self>) -> Option<usize> {
slf.inner.next()
}
}
#[pyclass]
struct Container {
iter: Vec<usize>,
}
#[pymethods]
impl Container {
fn __iter__(slf: PyRef<'_, Self>) -> PyResult<Py<Iter>> {
let iter = Iter {
inner: slf.iter.clone().into_iter(),
};
Py::new(slf.py(), iter)
}
}
# Python::attach(|py| {
# let container = Container { iter: vec![1, 2, 3, 4] };
# let inst = pyo3::Py::new(py, container).unwrap();
# pyo3::py_run!(py, inst, "assert list(inst) == [1, 2, 3, 4]");
# pyo3::py_run!(py, inst, "assert list(iter(iter(inst))) == [1, 2, 3, 4]");
# });
```
For more details on Python's iteration protocols, check out [the "Iterator Types" section of the library documentation](https://docs.python.org/library/stdtypes.html#iterator-types).
#### Returning a value from iteration
This guide has so far shown how to use `Option<T>` to implement yielding values during iteration.
In Python a generator can also return a value.
This is done by raising a `StopIteration` exception.
To express this in Rust, return `PyResult::Err` with a `PyStopIteration` as the error.
### Awaitable objects
- `__await__(<self>) -> object`
- `__aiter__(<self>) -> object`
- `__anext__(<self>) -> Option<object>`
### Mapping & Sequence types
The magic methods in this section can be used to implement Python container types.
They are two main categories of container in Python: "mappings" such as `dict`, with arbitrary keys, and "sequences" such as `list` and `tuple`, with integer keys.
The Python C-API which PyO3 is built upon has separate "slots" for sequences and mappings.
When writing a `class` in pure Python, there is no such distinction in the implementation - a `__getitem__` implementation will fill the slots for both the mapping and sequence forms, for example.
By default PyO3 reproduces the Python behaviour of filling both mapping and sequence slots.
This makes sense for the "simple" case which matches Python, and also for sequences, where the mapping slot is used anyway to implement slice indexing.
Mapping types usually will not want the sequence slots filled.
Having them filled will lead to outcomes which may be unwanted, such as:
- The mapping type will successfully cast to [`PySequence`].
This may lead to consumers of the type handling it incorrectly.
- Python provides a default implementation of `__iter__` for sequences, which calls `__getitem__` with consecutive positive integers starting from 0 until an `IndexError` is returned.
Unless the mapping only contains consecutive positive integer keys, this `__iter__` implementation will likely not be the intended behavior.
Use the `#[pyclass(mapping)]` annotation to instruct PyO3 to only fill the mapping slots, leaving the sequence ones empty.
This will apply to `__getitem__`, `__setitem__`, and `__delitem__`.
Use the `#[pyclass(sequence)]` annotation to instruct PyO3 to fill the `sq_length` slot instead of the `mp_length` slot for `__len__`.
This will help libraries such as `numpy` recognise the class as a sequence, however will also cause CPython to automatically add the sequence length to any negative indices before passing them to `__getitem__`. (`__getitem__`, `__setitem__` and `__delitem__` mapping slots are still used for sequences, for slice operations.)
- `__len__(<self>) -> usize`
Implements the built-in function `len()`.
<!-- rumdl-disable MD013 -->
<!-- TODO: report false positive -->
- `__contains__(<self>, object) -> bool`
Implements membership test operators.
Should return true if `item` is in `self`, false otherwise.
For objects that don’t define `__contains__()`, the membership test simply traverses the sequence until it finds a match.
<details>
<summary>Disabling Python's default contains</summary>
By default, all `#[pyclass]` types with an `__iter__` method support a
default implementation of the `in` operator. Types which do not want this
can override this by setting `__contains__` to `None`. This is the same
mechanism as for a pure-Python class. This is done like so:
```rust,no_run
# use pyo3::prelude::*;
#
#[pyclass]
struct NoContains {}
#[pymethods]
impl NoContains {
#[classattr]
const __contains__: Option<Py<PyAny>> = None;
}
```
</details>
<!-- rumdl-enable MD013 -->
- `__getitem__(<self>, object) -> object`
Implements retrieval of the `self[a]` element.
*Note:* Negative integer indexes are not handled specially by PyO3.
However, for classes with `#[pyclass(sequence)]`, when a negative index is accessed via `PySequence::get_item`, the underlying C API already adjusts the index to be positive.
- `__setitem__(<self>, object, object) -> ()`
Implements assignment to the `self[a]` element.
Should only be implemented if elements can be replaced.
Same behavior regarding negative indices as for `__getitem__`.
- `__delitem__(<self>, object) -> ()`
Implements deletion of the `self[a]` element.
Should only be implemented if elements can be deleted.
Same behavior regarding negative indices as for `__getitem__`.
- `fn __concat__(&self, other: impl FromPyObject) -> PyResult<impl ToPyObject>`
Concatenates two sequences.
Used by the `+` operator, after trying the numeric addition via the `__add__` and `__radd__` methods.
- `fn __repeat__(&self, count: isize) -> PyResult<impl ToPyObject>`
Repeats the sequence `count` times.
Used by the `*` operator, after trying the numeric multiplication via the `__mul__` and `__rmul__` methods.
- `fn __inplace_concat__(&self, other: impl FromPyObject) -> PyResult<impl ToPyObject>`
Concatenates two sequences.
Used by the `+=` operator, after trying the numeric addition via the `__iadd__` method.
- `fn __inplace_repeat__(&self, count: isize) -> PyResult<impl ToPyObject>`
Concatenates two sequences.
Used by the `*=` operator, after trying the numeric multiplication via the `__imul__` method.
### Descriptors
- `__get__(<self>, object, object) -> object`
- `__set__(<self>, object, object) -> ()`
- `__delete__(<self>, object) -> ()`
### Numeric types
Binary arithmetic operations (`+`, `-`, `*`, `@`, `/`, `//`, `%`, `divmod()`, `pow()` and `**`, `<<`, `>>`, `&`, `^`, and `|`) and their reflected versions:
(If the `object` is not of the type specified in the signature, the generated code will automatically `return NotImplemented`.)
- `__add__(<self>, object) -> object`
- `__radd__(<self>, object) -> object`
- `__sub__(<self>, object) -> object`
- `__rsub__(<self>, object) -> object`
- `__mul__(<self>, object) -> object`
- `__rmul__(<self>, object) -> object`
- `__matmul__(<self>, object) -> object`
- `__rmatmul__(<self>, object) -> object`
- `__floordiv__(<self>, object) -> object`
- `__rfloordiv__(<self>, object) -> object`
- `__truediv__(<self>, object) -> object`
- `__rtruediv__(<self>, object) -> object`
- `__divmod__(<self>, object) -> object`
- `__rdivmod__(<self>, object) -> object`
- `__mod__(<self>, object) -> object`
- `__rmod__(<self>, object) -> object`
- `__lshift__(<self>, object) -> object`
- `__rlshift__(<self>, object) -> object`
- `__rshift__(<self>, object) -> object`
- `__rrshift__(<self>, object) -> object`
- `__and__(<self>, object) -> object`
- `__rand__(<self>, object) -> object`
- `__xor__(<self>, object) -> object`
- `__rxor__(<self>, object) -> object`
- `__or__(<self>, object) -> object`
- `__ror__(<self>, object) -> object`
- `__pow__(<self>, object, object) -> object`
- `__rpow__(<self>, object, object) -> object`
In-place assignment operations (`+=`, `-=`, `*=`, `@=`, `/=`, `//=`, `%=`, `**=`, `<<=`, `>>=`, `&=`, `^=`, `|=`):
- `__iadd__(<self>, object) -> ()`
- `__isub__(<self>, object) -> ()`
- `__imul__(<self>, object) -> ()`
- `__imatmul__(<self>, object) -> ()`
- `__itruediv__(<self>, object) -> ()`
- `__ifloordiv__(<self>, object) -> ()`
- `__imod__(<self>, object) -> ()`
- `__ipow__(<self>, object, object) -> ()`
- `__ilshift__(<self>, object) -> ()`
- `__irshift__(<self>, object) -> ()`
- `__iand__(<self>, object) -> ()`
- `__ixor__(<self>, object) -> ()`
- `__ior__(<self>, object) -> ()`
Unary operations (`-`, `+`, `abs()` and `~`):
- `__pos__(<self>) -> object`
- `__neg__(<self>) -> object`
- `__abs__(<self>) -> object`
- `__invert__(<self>) -> object`
Coercions:
- `__index__(<self>) -> object (int)`
- `__int__(<self>) -> object (int)`
- `__float__(<self>) -> object (float)`
### Buffer objects
- `__getbuffer__(<self>, *mut ffi::Py_buffer, flags) -> ()`
- `__releasebuffer__(<self>, *mut ffi::Py_buffer) -> ()` Errors returned from `__releasebuffer__` will be sent to `sys.unraiseablehook`.
It is strongly advised to never return an error from `__releasebuffer__`, and if it really is necessary, to make best effort to perform any required freeing operations before returning.
`__releasebuffer__` will not be called a second time; anything not freed will be leaked.
### Garbage Collector Integration
If your type owns references to other Python objects, you will need to integrate with Python's garbage collector so that the GC is aware of those references.
To do this, implement the two methods `__traverse__` and `__clear__`.
These correspond to the slots `tp_traverse` and `tp_clear` in the Python C API.
`__traverse__` must call `visit.call()` for each reference to another Python object.
`__clear__` must clear out any mutable references to other Python objects (thus breaking reference cycles).
Immutable references do not have to be cleared, as every cycle must contain at least one mutable reference.
- `__traverse__(<self>, pyo3::class::gc::PyVisit<'_>) -> Result<(), pyo3::class::gc::PyTraverseError>`
- `__clear__(<self>) -> ()`
> [!NOTE]
> `__traverse__` does not work with [`#[pyo3(warn(...))]`](../function.md#warn).
Example:
```rust,no_run
use pyo3::prelude::*;
use pyo3::PyTraverseError;
use pyo3::gc::PyVisit;
#[pyclass]
struct ClassWithGCSupport {
obj: Option<Py<PyAny>>,
}
#[pymethods]
impl ClassWithGCSupport {
fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
visit.call(&self.obj)?;
Ok(())
}
fn __clear__(&mut self) {
// Clear reference, this decrements ref counter.
self.obj = None;
}
}
```
> [!NOTE]
> When a class inherits from either a Python builtins type or another type declared in Rust and implement either or both `__traverse__` and `__clear__`, the parent class `__traverse__` and `__clear__` is called automatically.
> There is no need to explicitly call it from inside the class implementation.
Usually, an implementation of `__traverse__` should do nothing but calls to `visit.call`.
Most importantly, safe access to the interpreter is prohibited inside implementations of `__traverse__`, i.e. `Python::attach` will panic.
> [!NOTE]
> These methods are part of the C API, PyPy does not necessarily honor them. If you are building for PyPy you should measure memory consumption to make sure you do not have runaway memory growth. See [this issue on the PyPy bug tracker](https://github.com/pypy/pypy/issues/3848).
[`PySequence`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PySequence.html
[`CompareOp::matches`]: {{#PYO3_DOCS_URL}}/pyo3/pyclass/enum.CompareOp.html#method.matches
+121
View File
@@ -0,0 +1,121 @@
# `#[pyclass]` thread safety
Python objects are freely shared between threads by the Python interpreter.
This means that:
- there is no control which thread might eventually drop the `#[pyclass]` object, meaning `Send` is required.
- multiple threads can potentially be reading the `#[pyclass]` data simultaneously, meaning `Sync` is required.
This section of the guide discusses various data structures which can be used to make types satisfy these requirements.
In special cases where it is known that your Python application is never going to use threads (this is rare!), these thread-safety requirements can be opted-out with [`#[pyclass(unsendable)]`](../class.md#customizing-the-class), at the cost of making concurrent access to the Rust data be runtime errors.
This is only for very specific use cases; it is almost always better to make proper thread-safe types.
## Making `#[pyclass]` types thread-safe
The general challenge with thread-safety is to make sure that two threads cannot produce a data race, i.e. unsynchronized writes to the same data at the same time.
A data race produces an unpredictable result and is forbidden by Rust.
By default, `#[pyclass]` employs an ["interior mutability" pattern](../class.md#bound-and-interior-mutability) to allow for either multiple `&T` references or a single exclusive `&mut T` reference to access the data.
This allows for simple `#[pyclass]` types to be thread-safe automatically, at the cost of runtime checking for concurrent access.
Errors will be raised if the usage overlaps.
For example, the below simple class is thread-safe:
```rust,no_run
# use pyo3::prelude::*;
#[pyclass]
struct MyClass {
x: i32,
y: i32,
}
#[pymethods]
impl MyClass {
fn get_x(&self) -> i32 {
self.x
}
fn set_y(&mut self, value: i32) {
self.y = value;
}
}
```
In the above example, if calls to `get_x` and `set_y` overlap (from two different threads) then at least one of those threads will experience a runtime error indicating that the data was "already borrowed".
To avoid these errors, you can take control of the interior mutability yourself in one of the following ways.
### Using atomic data structures
To remove the possibility of having overlapping `&self` and `&mut self` references produce runtime errors, consider using `#[pyclass(frozen)]` and use [atomic data structures](https://doc.rust-lang.org/std/sync/atomic/) to control modifications directly.
For example, a thread-safe version of the above `MyClass` using atomic integers would be as follows:
```rust,no_run
# use pyo3::prelude::*;
use std::sync::atomic::{AtomicI32, Ordering};
#[pyclass(frozen)]
struct MyClass {
x: AtomicI32,
y: AtomicI32,
}
#[pymethods]
impl MyClass {
fn get_x(&self) -> i32 {
self.x.load(Ordering::Relaxed)
}
fn set_y(&self, value: i32) {
self.y.store(value, Ordering::Relaxed)
}
}
```
### Using locks
An alternative to atomic data structures is to use [locks](https://doc.rust-lang.org/std/sync/struct.Mutex.html) to make threads wait for access to shared data.
For example, a thread-safe version of the above `MyClass` using locks would be as follows:
```rust,no_run
# use pyo3::prelude::*;
use std::sync::Mutex;
struct MyClassInner {
x: i32,
y: i32,
}
#[pyclass(frozen)]
struct MyClass {
inner: Mutex<MyClassInner>
}
#[pymethods]
impl MyClass {
fn get_x(&self) -> i32 {
self.inner.lock().expect("lock not poisoned").x
}
fn set_y(&self, value: i32) {
self.inner.lock().expect("lock not poisoned").y = value;
}
}
```
If you need to lock around state stored in the Python interpreter or otherwise call into the Python C API while a lock is held, you might find the `MutexExt` trait useful.
It provides a `lock_py_attached` method for `std::sync::Mutex` that avoids deadlocks with the GIL or other global synchronization events in the interpreter.
Additionally, support for the `parking_lot` and `lock_api` synchronization libraries is gated behind the `parking_lot` and `lock_api` features.
You can also enable the `arc_lock` feature if you need the `arc_lock` features of either library.
### Wrapping unsynchronized data
In some cases, the data structures stored within a `#[pyclass]` may themselves not be thread-safe.
Rust will therefore not implement `Send` and `Sync` on the `#[pyclass]` type.
To achieve thread-safety, a manual `Send` and `Sync` implementation is required which is `unsafe` and should only be done following careful review of the soundness of the implementation.
Doing this for PyO3 types is no different than for any other Rust code, [the Rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html) has a great discussion on this.
+1
View File
@@ -0,0 +1 @@
{{#include ../../Contributing.md}}
+5
View File
@@ -0,0 +1,5 @@
# Type conversions
In this portion of the guide we'll talk about the mapping of Python types to Rust types offered by PyO3, as well as the traits available to perform conversions between them.
See also the conversion [tables](conversions/tables.md) and [traits](conversions/traits.md).
+129
View File
@@ -0,0 +1,129 @@
# Mapping of Rust types to Python types
When writing functions callable from Python (such as a `#[pyfunction]` or in a `#[pymethods]` block), the trait `FromPyObject` is required for function arguments, and `IntoPyObject` is required for function return values.
Consult the tables in the following section to find the Rust types provided by PyO3 which implement these traits.
## Argument Types
When accepting a function argument, it is possible to either use Rust library types or PyO3's Python-native types. (See the next section for discussion on when to use each.)
The table below contains the Python type and the corresponding function argument types that will accept them:
| Python | Rust | Rust (Python-native) |
| ------------- |:-------------------------------:|:--------------------:|
| `object` | - | `PyAny` |
| `str` | `String`, `Cow<str>`, `&str`, `char`, `OsString`, `PathBuf`, `Path` | `PyString` |
| `bytes` | `Vec<u8>`, `&[u8]`, `Cow<[u8]>` | `PyBytes` |
| `bool` | `bool` | `PyBool` |
| `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `PyInt` |
| `float` | `f32`, `f64`, `ordered_float::NotNan`[^2], `ordered_float::OrderedFloat`[^2] | `PyFloat` |
| `complex` | `num_complex::Complex`[^3] | `PyComplex` |
| `fractions.Fraction`| `num_rational::Ratio`[^4] | - |
| `list[T]` | `Vec<T>` | `PyList` |
| `dict[K, V]` | `HashMap<K, V>`, `BTreeMap<K, V>`, `hashbrown::HashMap<K, V>`[^5], `indexmap::IndexMap<K, V>`[^6] | `PyDict` |
| `tuple[T, U]` | `(T, U)`, `Vec<T>` | `PyTuple` |
| `set[T]` | `HashSet<T>`, `BTreeSet<T>`, `hashbrown::HashSet<T>`[^5] | `PySet` |
| `frozenset[T]` | `HashSet<T>`, `BTreeSet<T>`, `hashbrown::HashSet<T>`[^5] | `PyFrozenSet` |
| `bytearray` | `Vec<u8>`, `Cow<[u8]>` | `PyByteArray` |
| `slice` | - | `PySlice` |
| `type` | - | `PyType` |
| `module` | - | `PyModule` |
| `collections.abc.Buffer` | - | `PyBuffer<T>` |
| `datetime.datetime` | `SystemTime`, `chrono::DateTime<Tz>`[^7], `chrono::NaiveDateTime`[^7] | `PyDateTime` |
| `datetime.date` | `chrono::NaiveDate`[^7] | `PyDate` |
| `datetime.time` | `chrono::NaiveTime`[^7] | `PyTime` |
| `datetime.tzinfo` | `chrono::FixedOffset`[^7], `chrono::Utc`[^7], `chrono_tz::TimeZone`[^8] | `PyTzInfo` |
| `datetime.timedelta` | `Duration`, `chrono::Duration`[^7] | `PyDelta` |
| `decimal.Decimal` | `rust_decimal::Decimal`[^9] | - |
| `decimal.Decimal` | `bigdecimal::BigDecimal`[^10] | - |
| `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::Ipv4Addr` | - |
| `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::Ipv6Addr` | - |
| `os.PathLike` | `PathBuf`, `Path` | `PyString` |
| `pathlib.Path` | `PathBuf`, `Path` | `PyString` |
| `typing.Optional[T]` | `Option<T>` | - |
| `typing.Sequence[T]` | `Vec<T>` | `PySequence` |
| `typing.Mapping[K, V]` | `HashMap<K, V>`, `BTreeMap<K, V>`, `hashbrown::HashMap<K, V>`[^5], `indexmap::IndexMap<K, V>`[^6] | `&PyMapping` |
| `typing.Iterator[Any]` | - | `PyIterator` |
| `typing.Union[...]` | See [`#[derive(FromPyObject)]`](traits.md#deriving-frompyobject-for-enums) | - |
It is also worth remembering the following special types:
| What | Description |
| ---------------- | ------------------------------------- |
| `Python<'py>` | A token used to prove attachment to the Python interpreter. |
| `Bound<'py, T>` | A Python object with a lifetime which binds it to the attachment to the Python interpreter. This provides access to most of PyO3's APIs. |
| `Py<T>` | A Python object not connected to any lifetime of attachment to the Python interpreter. This can be sent to other threads. |
| `PyRef<T>` | A `#[pyclass]` borrowed immutably. |
| `PyRefMut<T>` | A `#[pyclass]` borrowed mutably. |
For more detail on accepting `#[pyclass]` values as function arguments, see [the section of this guide on Python Classes](../class.md).
### Using Rust library types vs Python-native types
Using Rust library types as function arguments will incur a conversion cost compared to using the Python-native types.
Using the Python-native types is almost zero-cost (they just require a type check similar to the Python builtin function `isinstance()`).
However, once that conversion cost has been paid, the Rust standard library types offer a number of benefits:
- You can write functionality in native-speed Rust code (free of Python's runtime costs).
- You get better interoperability with the rest of the Rust ecosystem.
- You can use `Python::detach` to detach from the interpreter and let other Python threads make progress while your Rust code is executing.
- You also benefit from stricter type checking.
For example you can specify `Vec<i32>`, which will only accept a Python `list` containing integers.
The Python-native equivalent, `&PyList`, would accept a Python `list` containing Python objects of any type.
For most PyO3 usage the conversion cost is worth paying to get these benefits.
As always, if you're not sure it's worth it in your case, benchmark it!
## Returning Rust values to Python
When returning values from functions callable from Python, PyO3's smart pointers (`Py<T>`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`) can be used with zero cost.
Because `Bound<'py, T>` and `Borrowed<'a, 'py, T>` have lifetime parameters, the Rust compiler may ask for lifetime annotations to be added to your function.
See the [section of the guide dedicated to this](../types.md#function-argument-lifetimes).
If your function is fallible, it should return `PyResult<T>` or `Result<T, E>` where `E` implements `From<E> for PyErr`.
This will raise a `Python` exception if the `Err` variant is returned.
Finally, the following Rust types are also able to convert to Python as return values:
| Rust type | Resulting Python Type |
| ------------- |:-------------------------------:|
| `String` | `str` |
| `&str` | `str` |
| `bool` | `bool` |
| Any integer type (`i32`, `u32`, `usize`, etc) | `int` |
| `f32`, `f64` | `float` |
| `Option<T>` | `Optional[T]` |
| `(T, U)` | `Tuple[T, U]` |
| `Vec<T>` | `List[T]` |
| `Cow<[u8]>` | `bytes` |
| `HashMap<K, V>` | `Dict[K, V]` |
| `BTreeMap<K, V>` | `Dict[K, V]` |
| `HashSet<T>` | `Set[T]` |
| `BTreeSet<T>` | `Set[T]` |
| `Py<T>` | `T` |
| `Bound<T>` | `T` |
| `PyRef<T: PyClass>` | `T` |
| `PyRefMut<T: PyClass>` | `T` |
[^1]: Requires the `num-bigint` optional feature.
[^2]: Requires the `ordered-float` optional feature.
[^3]: Requires the `num-complex` optional feature.
[^4]: Requires the `num-rational` optional feature.
[^5]: Requires the `hashbrown` optional feature.
[^6]: Requires the `indexmap` optional feature.
[^7]: Requires the `chrono` (and maybe `chrono-local`) optional feature(s).
[^8]: Requires the `chrono-tz` optional feature.
[^9]: Requires the `rust_decimal` optional feature.
[^10]: Requires the `bigdecimal` optional feature.
+763
View File
@@ -0,0 +1,763 @@
# Conversion traits
PyO3 provides some handy traits to convert between Python types and Rust types.
## `.extract()` and the `FromPyObject` trait
The easiest way to convert a Python object to a Rust value is using `.extract()`.
It returns a `PyResult` with a type error if the conversion fails, so usually you will use something like
```rust
# use pyo3::prelude::*;
# use pyo3::types::PyList;
# fn main() -> PyResult<()> {
# Python::attach(|py| {
# let list = PyList::new(py, b"foo")?;
let v: Vec<i32> = list.extract()?;
# assert_eq!(&v, &[102, 111, 111]);
# Ok(())
# })
# }
```
This method is available for many Python object types, and can produce a wide variety of Rust types, which you can check out in the implementor list of [`FromPyObject`].
[`FromPyObject`] is also implemented for your own Rust types wrapped as Python objects (see [the chapter about classes](../class.md)).
There, in order to both be able to operate on mutable references *and* satisfy Rust's rules of non-aliasing mutable references, you have to extract the PyO3 reference wrappers [`PyRef`] and [`PyRefMut`].
They work like the reference wrappers of `std::cell::RefCell` and ensure (at runtime) that Rust borrows are allowed.
### Deriving [`FromPyObject`]
[`FromPyObject`] can be automatically derived for many kinds of structs and enums if the member types themselves implement `FromPyObject`.
This even includes members with a generic type `T: FromPyObject`.
Derivation for empty enums, enum variants and structs is not supported.
### Deriving [`FromPyObject`] for structs
The derivation generates code that will attempt to access the attribute `my_string` on the Python object, i.e. `obj.getattr("my_string")`, and call `extract()` on the attribute.
```rust
use pyo3::prelude::*;
#[derive(FromPyObject)]
struct RustyStruct {
my_string: String,
}
#
# fn main() -> PyResult<()> {
# Python::attach(|py| -> PyResult<()> {
# let module = PyModule::from_code(
# py,
# c"class Foo:
# def __init__(self):
# self.my_string = 'test'",
# c"<string>",
# c"",
# )?;
#
# let class = module.getattr("Foo")?;
# let instance = class.call0()?;
# let rustystruct: RustyStruct = instance.extract()?;
# assert_eq!(rustystruct.my_string, "test");
# Ok(())
# })
# }
```
By setting the `#[pyo3(item)]` attribute on the field, PyO3 will attempt to extract the value by calling the `get_item` method on the Python object.
```rust
use pyo3::prelude::*;
#[derive(FromPyObject)]
struct RustyStruct {
#[pyo3(item)]
my_string: String,
}
#
# use pyo3::types::PyDict;
# fn main() -> PyResult<()> {
# Python::attach(|py| -> PyResult<()> {
# let dict = PyDict::new(py);
# dict.set_item("my_string", "test")?;
#
# let rustystruct: RustyStruct = dict.extract()?;
# assert_eq!(rustystruct.my_string, "test");
# Ok(())
# })
# }
```
The argument passed to `getattr` and `get_item` can also be configured:
```rust
use pyo3::prelude::*;
#[derive(FromPyObject)]
struct RustyStruct {
#[pyo3(item("key"))]
string_in_mapping: String,
#[pyo3(attribute("name"))]
string_attr: String,
}
#
# fn main() -> PyResult<()> {
# Python::attach(|py| -> PyResult<()> {
# let module = PyModule::from_code(
# py,
# c"class Foo(dict):
# def __init__(self):
# self.name = 'test'
# self['key'] = 'test2'",
# c"<string>",
# c"",
# )?;
#
# let class = module.getattr("Foo")?;
# let instance = class.call0()?;
# let rustystruct: RustyStruct = instance.extract()?;
# assert_eq!(rustystruct.string_attr, "test");
# assert_eq!(rustystruct.string_in_mapping, "test2");
#
# Ok(())
# })
# }
```
This tries to extract `string_attr` from the attribute `name` and `string_in_mapping` from a mapping with the key `"key"`.
The arguments for `attribute` are restricted to non-empty string literals while `item` can take any valid literal that implements `ToBorrowedObject`.
You can use `#[pyo3(from_item_all)]` on a struct to extract every field with `get_item` method.
In this case, you can't use `#[pyo3(attribute)]` or barely use `#[pyo3(item)]` on any field.
However, using `#[pyo3(item("key"))]` to specify the key for a field is still allowed.
```rust
use pyo3::prelude::*;
#[derive(FromPyObject)]
#[pyo3(from_item_all)]
struct RustyStruct {
foo: String,
bar: String,
#[pyo3(item("foobar"))]
baz: String,
}
#
# fn main() -> PyResult<()> {
# Python::attach(|py| -> PyResult<()> {
# let py_dict = py.eval(c"{'foo': 'foo', 'bar': 'bar', 'foobar': 'foobar'}", None, None)?;
# let rustystruct: RustyStruct = py_dict.extract()?;
# assert_eq!(rustystruct.foo, "foo");
# assert_eq!(rustystruct.bar, "bar");
# assert_eq!(rustystruct.baz, "foobar");
#
# Ok(())
# })
# }
```
### Deriving [`FromPyObject`] for tuple structs
Tuple structs are also supported but do not allow customizing the extraction.
The input is always assumed to be a Python tuple with the same length as the Rust type, the `n`th field is extracted from the `n`th item in the Python tuple.
```rust
use pyo3::prelude::*;
#[derive(FromPyObject)]
struct RustyTuple(String, String);
# use pyo3::types::PyTuple;
# fn main() -> PyResult<()> {
# Python::attach(|py| -> PyResult<()> {
# let tuple = PyTuple::new(py, vec!["test", "test2"])?;
#
# let rustytuple: RustyTuple = tuple.extract()?;
# assert_eq!(rustytuple.0, "test");
# assert_eq!(rustytuple.1, "test2");
#
# Ok(())
# })
# }
```
Tuple structs with a single field are treated as wrapper types which are described in the following section.
To override this behaviour and ensure that the input is in fact a tuple, specify the struct as
```rust
use pyo3::prelude::*;
#[derive(FromPyObject)]
struct RustyTuple((String,));
# use pyo3::types::PyTuple;
# fn main() -> PyResult<()> {
# Python::attach(|py| -> PyResult<()> {
# let tuple = PyTuple::new(py, vec!["test"])?;
#
# let rustytuple: RustyTuple = tuple.extract()?;
# assert_eq!((rustytuple.0).0, "test");
#
# Ok(())
# })
# }
```
### Deriving [`FromPyObject`] for wrapper types
The `pyo3(transparent)` attribute can be used on structs with exactly one field.
This results in extracting directly from the input object, i.e. `obj.extract()`, rather than trying to access an item or attribute.
This behaviour is enabled per default for newtype structs and tuple-variants with a single field.
```rust
use pyo3::prelude::*;
#[derive(FromPyObject)]
struct RustyTransparentTupleStruct(String);
#[derive(FromPyObject)]
#[pyo3(transparent)]
struct RustyTransparentStruct {
inner: String,
}
# use pyo3::types::PyString;
# fn main() -> PyResult<()> {
# Python::attach(|py| -> PyResult<()> {
# let s = PyString::new(py, "test");
#
# let tup: RustyTransparentTupleStruct = s.extract()?;
# assert_eq!(tup.0, "test");
#
# let stru: RustyTransparentStruct = s.extract()?;
# assert_eq!(stru.inner, "test");
#
# Ok(())
# })
# }
```
### Deriving [`FromPyObject`] for enums
The `FromPyObject` derivation for enums generates code that tries to extract the variants in the order of the fields.
As soon as a variant can be extracted successfully, that variant is returned.
This makes it possible to extract Python union types like `str | int`.
The same customizations and restrictions described for struct derivations apply to enum variants, i.e. a tuple variant assumes that the input is a Python tuple, and a struct variant defaults to extracting fields as attributes but can be configured in the same manner.
The `transparent` attribute can be applied to single-field-variants.
```rust
use pyo3::prelude::*;
#[derive(FromPyObject)]
# #[derive(Debug)]
enum RustyEnum<'py> {
Int(usize), // input is a positive int
String(String), // input is a string
IntTuple(usize, usize), // input is a 2-tuple with positive ints
StringIntTuple(String, usize), // input is a 2-tuple with String and int
Coordinates3d {
// needs to be in front of 2d
x: usize,
y: usize,
z: usize,
},
Coordinates2d {
// only gets checked if the input did not have `z`
#[pyo3(attribute("x"))]
a: usize,
#[pyo3(attribute("y"))]
b: usize,
},
#[pyo3(transparent)]
CatchAll(Bound<'py, PyAny>), // This extraction never fails
}
#
# use pyo3::types::{PyBytes, PyString};
# fn main() -> PyResult<()> {
# Python::attach(|py| -> PyResult<()> {
# {
# let thing = 42_u8.into_pyobject(py)?;
# let rust_thing: RustyEnum<'_> = thing.extract()?;
#
# assert_eq!(
# 42,
# match rust_thing {
# RustyEnum::Int(i) => i,
# other => unreachable!("Error extracting: {:?}", other),
# }
# );
# }
# {
# let thing = PyString::new(py, "text");
# let rust_thing: RustyEnum<'_> = thing.extract()?;
#
# assert_eq!(
# "text",
# match rust_thing {
# RustyEnum::String(i) => i,
# other => unreachable!("Error extracting: {:?}", other),
# }
# );
# }
# {
# let thing = (32_u8, 73_u8).into_pyobject(py)?;
# let rust_thing: RustyEnum<'_> = thing.extract()?;
#
# assert_eq!(
# (32, 73),
# match rust_thing {
# RustyEnum::IntTuple(i, j) => (i, j),
# other => unreachable!("Error extracting: {:?}", other),
# }
# );
# }
# {
# let thing = ("foo", 73_u8).into_pyobject(py)?;
# let rust_thing: RustyEnum<'_> = thing.extract()?;
#
# assert_eq!(
# (String::from("foo"), 73),
# match rust_thing {
# RustyEnum::StringIntTuple(i, j) => (i, j),
# other => unreachable!("Error extracting: {:?}", other),
# }
# );
# }
# {
# let module = PyModule::from_code(
# py,
# c"class Foo(dict):
# def __init__(self):
# self.x = 0
# self.y = 1
# self.z = 2",
# c"<string>",
# c"",
# )?;
#
# let class = module.getattr("Foo")?;
# let instance = class.call0()?;
# let rust_thing: RustyEnum<'_> = instance.extract()?;
#
# assert_eq!(
# (0, 1, 2),
# match rust_thing {
# RustyEnum::Coordinates3d { x, y, z } => (x, y, z),
# other => unreachable!("Error extracting: {:?}", other),
# }
# );
# }
#
# {
# let module = PyModule::from_code(
# py,
# c"class Foo(dict):
# def __init__(self):
# self.x = 3
# self.y = 4",
# c"<string>",
# c"",
# )?;
#
# let class = module.getattr("Foo")?;
# let instance = class.call0()?;
# let rust_thing: RustyEnum<'_> = instance.extract()?;
#
# assert_eq!(
# (3, 4),
# match rust_thing {
# RustyEnum::Coordinates2d { a, b } => (a, b),
# other => unreachable!("Error extracting: {:?}", other),
# }
# );
# }
#
# {
# let thing = PyBytes::new(py, b"text");
# let rust_thing: RustyEnum<'_> = thing.extract()?;
#
# assert_eq!(
# b"text",
# match rust_thing {
# RustyEnum::CatchAll(ref i) => i.cast::<PyBytes>()?.as_bytes(),
# other => unreachable!("Error extracting: {:?}", other),
# }
# );
# }
# Ok(())
# })
# }
```
If none of the enum variants match, a `PyTypeError` containing the names of the tested variants is returned.
The names reported in the error message can be customized through the `#[pyo3(annotation = "name")]` attribute, e.g. to use conventional Python type names:
```rust
use pyo3::prelude::*;
#[derive(FromPyObject)]
# #[derive(Debug)]
enum RustyEnum {
#[pyo3(transparent, annotation = "str")]
String(String),
#[pyo3(transparent, annotation = "int")]
Int(isize),
}
#
# fn main() -> PyResult<()> {
# Python::attach(|py| -> PyResult<()> {
# {
# let thing = 42_u8.into_pyobject(py)?;
# let rust_thing: RustyEnum = thing.extract()?;
#
# assert_eq!(
# 42,
# match rust_thing {
# RustyEnum::Int(i) => i,
# other => unreachable!("Error extracting: {:?}", other),
# }
# );
# }
#
# {
# let thing = "foo".into_pyobject(py)?;
# let rust_thing: RustyEnum = thing.extract()?;
#
# assert_eq!(
# "foo",
# match rust_thing {
# RustyEnum::String(i) => i,
# other => unreachable!("Error extracting: {:?}", other),
# }
# );
# }
#
# {
# let thing = b"foo".into_pyobject(py)?;
# let error = thing.extract::<RustyEnum>().unwrap_err();
# assert!(error.is_instance_of::<pyo3::exceptions::PyTypeError>(py));
# }
#
# Ok(())
# })
# }
```
If the input is neither a string nor an integer, the error message will be: `"'<INPUT_TYPE>' is not an instance of 'str | int'"`.
### `#[derive(FromPyObject)]` Container Attributes
- `pyo3(transparent)`
- extract the field directly from the object as `obj.extract()` instead of `get_item()` or
`getattr()`
- Newtype structs and tuple-variants are treated as transparent per default.
- only supported for single-field structs and enum variants
- `pyo3(annotation = "name")`
- changes the name of the failed variant in the generated error message in case of failure.
- e.g. `pyo3("int")` reports the variant's type as `int`.
- only supported for enum variants
- `pyo3(rename_all = "...")`
- renames all attributes/item keys according to the specified renaming rule
- Possible values are: "camelCase", "kebab-case", "lowercase", "PascalCase", "SCREAMING-KEBAB-CASE", "SCREAMING_SNAKE_CASE", "snake_case", "UPPERCASE".
- fields with an explicit renaming via `attribute(...)`/`item(...)` are not affected
### `#[derive(FromPyObject)]` Field Attributes
- `pyo3(attribute)`, `pyo3(attribute("name"))`
- retrieve the field from an attribute, possibly with a custom name specified as an argument
- argument must be a string-literal.
- `pyo3(item)`, `pyo3(item("key"))`
- retrieve the field from a mapping, possibly with the custom key specified as an argument.
- can be any literal that implements `ToBorrowedObject`
- `pyo3(from_py_with = ...)`
- apply a custom function to convert the field from Python the desired Rust type.
- the argument must be the path to the function.
- the function signature must be `fn(&Bound<PyAny>) -> PyResult<T>` where `T` is the Rust type of the argument.
- `pyo3(default)`, `pyo3(default = ...)`
- if the argument is set, uses the given default value.
- in this case, the argument must be a Rust expression returning a value of the desired Rust type.
- if the argument is not set, [`Default::default`](https://doc.rust-lang.org/std/default/trait.Default.html#tymethod.default) is used.
- note that the default value is only used if the field is not set.
If the field is set and the conversion function from Python to Rust fails, an exception is raised and the default value is not used.
- this attribute is only supported on named fields.
For example, the code below applies the given conversion function on the `"value"` dict item to compute its length or fall back to the type default value (0):
```rust
use pyo3::prelude::*;
#[derive(FromPyObject)]
struct RustyStruct {
#[pyo3(item("value"), default, from_py_with = Bound::<'_, PyAny>::len)]
len: usize,
#[pyo3(item)]
other: usize,
}
#
# use pyo3::types::PyDict;
# fn main() -> PyResult<()> {
# Python::attach(|py| -> PyResult<()> {
# // Filled case
# let dict = PyDict::new(py);
# dict.set_item("value", (1,)).unwrap();
# dict.set_item("other", 1).unwrap();
# let result = dict.extract::<RustyStruct>()?;
# assert_eq!(result.len, 1);
# assert_eq!(result.other, 1);
#
# // Empty case
# let dict = PyDict::new(py);
# dict.set_item("other", 1).unwrap();
# let result = dict.extract::<RustyStruct>()?;
# assert_eq!(result.len, 0);
# assert_eq!(result.other, 1);
# Ok(())
# })
# }
```
### ⚠ Phase-Out of `FromPyObject` blanket implementation for cloneable PyClasses ⚠
Historically PyO3 has provided a blanket implementation for `#[pyclass]` types that also implement `Clone`, to allow extraction of such types by value.
Over time this has turned out problematic for a few reasons, the major one being the prevention of custom conversions by downstream crates if their type is `Clone`.
Over the next few releases the blanket implementation is gradually phased out, and eventually replaced by an opt-in option.
As a first step of this migration a new `skip_from_py_object` option for `#[pyclass]` was introduced, to opt-out of the blanket implementation and allow downstream users to provide their own implementation:
```rust
# #![allow(dead_code)]
# use pyo3::prelude::*;
#[pyclass(skip_from_py_object)] // opt-out of the PyO3 FromPyObject blanket
#[derive(Clone)]
struct Number(i32);
impl<'py> FromPyObject<'_, 'py> for Number {
type Error = PyErr;
fn extract(obj: pyo3::Borrowed<'_, 'py, pyo3::PyAny>) -> Result<Self, Self::Error> {
if let Ok(obj) = obj.cast::<Self>() { // first try extraction via class object
Ok(obj.borrow().clone())
} else {
obj.extract::<i32>().map(Self) // otherwise try integer directly
}
}
}
```
As a second step the `from_py_object` option was introduced.
This option also opts-out of the blanket implementation and instead generates a custom `FromPyObject` implementation for the pyclass which is functionally equivalent to the blanket.
## `IntoPyObject`
The [`IntoPyObject`] trait defines the to-python conversion for a Rust type.
All types in PyO3 implement this trait, as does a `#[pyclass]` which doesn't use `extends`.
This trait defines a single method, `into_pyobject()`, which returns a [`Result`] with `Ok` and `Err` types depending on the input value.
For convenience, there is a companion [`IntoPyObjectExt`] trait which adds methods such as `into_py_any()` which converts the `Ok` and `Err` types to commonly used types (in the case of `into_py_any()`, `Py<PyAny>` and `PyErr` respectively).
Occasionally you may choose to implement this for custom types which are mapped to Python types *without* having a unique python type.
### derive macro
`IntoPyObject` can be implemented using our derive macro.
Both `struct`s and `enum`s are supported.
`struct`s will turn into a `PyDict` using the field names as keys, tuple `struct`s will turn convert into `PyTuple` with the fields in declaration order.
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
# use std::collections::HashMap;
# use std::hash::Hash;
// structs convert into `PyDict` with field names as keys
#[derive(IntoPyObject)]
struct Struct {
count: usize,
obj: Py<PyAny>,
}
// tuple structs convert into `PyTuple`
// lifetimes and generics are supported, the impl will be bounded by
// `K: IntoPyObject, V: IntoPyObject`
#[derive(IntoPyObject)]
struct Tuple<'a, K: Hash + Eq, V>(&'a str, HashMap<K, V>);
```
For structs with a single field (newtype pattern) the `#[pyo3(transparent)]` option can be used to forward the implementation to the inner type.
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
// newtype tuple structs are implicitly `transparent`
#[derive(IntoPyObject)]
struct TransparentTuple(Py<PyAny>);
#[derive(IntoPyObject)]
#[pyo3(transparent)]
struct TransparentStruct<'py> {
inner: Bound<'py, PyAny>, // `'py` lifetime will be used as the Python lifetime
}
```
For `enum`s each variant is converted according to the rules for `struct`s above.
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
# use std::collections::HashMap;
# use std::hash::Hash;
#[derive(IntoPyObject)]
enum Enum<'a, 'py, K: Hash + Eq, V> { // enums are supported and convert using the same
TransparentTuple(Py<PyAny>), // rules on the variants as the structs above
#[pyo3(transparent)]
TransparentStruct { inner: Bound<'py, PyAny> },
Tuple(&'a str, HashMap<K, V>),
Struct { count: usize, obj: Py<PyAny> }
}
```
Additionally `IntoPyObject` can be derived for a reference to a struct or enum using the `IntoPyObjectRef` derive macro.
All the same rules from above apply as well.
#### `#[derive(IntoPyObject)]`/`#[derive(IntoPyObjectRef)]` Field Attributes
- `pyo3(into_py_with = ...)`
- apply a custom function to convert the field from Rust into Python.
- the argument must be the function identifier
- the function signature must be `fn(Cow<'_, T>, Python<'py>) -> PyResult<Bound<'py, PyAny>>` where `T` is the Rust type of the argument.
- `#[derive(IntoPyObject)]` will invoke the function with `Cow::Owned`
- `#[derive(IntoPyObjectRef)]` will invoke the function with `Cow::Borrowed`
```rust,no_run
# use pyo3::prelude::*;
# use pyo3::IntoPyObjectExt;
# use std::borrow::Cow;
#[derive(Clone)]
struct NotIntoPy(usize);
#[derive(IntoPyObject, IntoPyObjectRef)]
struct MyStruct {
#[pyo3(into_py_with = convert)]
not_into_py: NotIntoPy,
}
/// Convert `NotIntoPy` into Python
fn convert<'py>(not_into_py: Cow<'_, NotIntoPy>, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
not_into_py.0.into_bound_py_any(py)
}
```
### manual implementation
If the derive macro is not suitable for your use case, `IntoPyObject` can be implemented manually as demonstrated below.
```rust,no_run
# use pyo3::prelude::*;
# #[allow(dead_code)]
struct MyPyObjectWrapper(Py<PyAny>);
impl<'py> IntoPyObject<'py> for MyPyObjectWrapper {
type Target = PyAny; // the Python type
type Output = Bound<'py, Self::Target>; // in most cases this will be `Bound`
type Error = std::convert::Infallible; // the conversion error type, has to be convertible to `PyErr`
fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
Ok(self.0.into_bound(py))
}
}
// equivalent to former `ToPyObject` implementations
impl<'a, 'py> IntoPyObject<'py> for &'a MyPyObjectWrapper {
type Target = PyAny;
type Output = Borrowed<'a, 'py, Self::Target>; // `Borrowed` can be used to optimized reference counting
type Error = std::convert::Infallible;
fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
Ok(self.0.bind_borrowed(py))
}
}
```
### `BoundObject` for conversions that may be `Bound` or `Borrowed`
`IntoPyObject::into_py_object` returns either `Bound` or `Borrowed` depending on the implementation for a concrete type.
For example, the `IntoPyObject` implementation for `u32` produces a `Bound<'py, PyInt>` and the `bool` implementation produces a `Borrowed<'py, 'py, PyBool>`:
```rust,no_run
use pyo3::prelude::*;
use pyo3::IntoPyObject;
use pyo3::types::{PyBool, PyInt};
let ints: Vec<u32> = vec![1, 2, 3, 4];
let bools = vec![true, false, false, true];
Python::attach(|py| {
let ints_as_pyint: Vec<Bound<'_, PyInt>> = ints
.iter()
.map(|x| Ok(x.into_pyobject(py)?))
.collect::<PyResult<_>>()
.unwrap();
let bools_as_pybool: Vec<Borrowed<'_, '_, PyBool>> = bools
.iter()
.map(|x| Ok(x.into_pyobject(py)?))
.collect::<PyResult<_>>()
.unwrap();
});
```
In this example if we wanted to combine `ints_as_pyints` and `bools_as_pybool` into a single `Vec<Py<PyAny>>` to return from the `Python::attach` closure, we would have to manually convert the concrete types for the smart pointers and the python types.
Instead, we can write a function that generically converts vectors of either integers or bools into a vector of `Py<PyAny>` using the [`BoundObject`] trait:
```rust,no_run
# use pyo3::prelude::*;
# use pyo3::BoundObject;
# use pyo3::IntoPyObject;
# let bools = vec![true, false, false, true];
# let ints = vec![1, 2, 3, 4];
fn convert_to_vec_of_pyobj<'py, T>(py: Python<'py>, the_vec: Vec<T>) -> PyResult<Vec<Py<PyAny>>>
where
T: IntoPyObject<'py> + Copy
{
the_vec.iter()
.map(|x| {
Ok(
// Note: the below is equivalent to `x.into_py_any()`
// from the `IntoPyObjectExt` trait
x.into_pyobject(py)
.map_err(Into::into)?
.into_any()
.unbind()
)
})
.collect()
}
let vec_of_pyobjs: Vec<Py<PyAny>> = Python::attach(|py| {
let mut bools_as_pyany = convert_to_vec_of_pyobj(py, bools).unwrap();
let mut ints_as_pyany = convert_to_vec_of_pyobj(py, ints).unwrap();
let mut result: Vec<Py<PyAny>> = vec![];
result.append(&mut bools_as_pyany);
result.append(&mut ints_as_pyany);
result
});
```
In the example above we used `BoundObject::into_any` and `BoundObject::unbind` to manipulate the python types and smart pointers into the result type we wanted to produce from the function.
[`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html
[`IntoPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.IntoPyObject.html
[`IntoPyObjectExt`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.IntoPyObjectExt.html
[`PyRef`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRef.html
[`PyRefMut`]: {{#PYO3_DOCS_URL}}/pyo3/pycell/struct.PyRefMut.html
[`BoundObject`]: {{#PYO3_DOCS_URL}}/pyo3/instance/trait.BoundObject.html
[`Result`]: https://doc.rust-lang.org/stable/std/result/enum.Result.html
+381
View File
@@ -0,0 +1,381 @@
# Debugging
## Macros
PyO3's attributes (`#[pyclass]`, `#[pymodule]`, etc.) are [procedural macros](https://doc.rust-lang.org/reference/procedural-macros.html), which means that they rewrite the source of the annotated item.
You can view the generated source with the following command, which also expands a few other things:
```bash
cargo rustc --profile=check -- -Z unstable-options --pretty=expanded > expanded.rs; rustfmt expanded.rs
```
(You might need to install [rustfmt](https://github.com/rust-lang-nursery/rustfmt) if you don't already have it.)
You can also debug classic `!`-macros by adding `-Z trace-macros`:
```bash
cargo rustc --profile=check -- -Z unstable-options --pretty=expanded -Z trace-macros > expanded.rs; rustfmt expanded.rs
```
Note that those commands require using the nightly build of rust and may occasionally have bugs.
See [cargo expand](https://github.com/dtolnay/cargo-expand) for a more elaborate and stable version of those commands.
## Running with Valgrind
Valgrind is a tool to detect memory management bugs such as memory leaks.
You first need to install a debug build of Python, otherwise Valgrind won't produce usable results.
In Ubuntu there's e.g. a `python3-dbg` package.
Activate an environment with the debug interpreter and recompile.
If you're on Linux, use `ldd` with the name of your binary and check that you're linking e.g. `libpython3.8d.so.1.0` instead of `libpython3.8.so.1.0`.
[Download the suppressions file for CPython](https://raw.githubusercontent.com/python/cpython/master/Misc/valgrind-python.supp).
Run Valgrind with `valgrind --suppressions=valgrind-python.supp ./my-command --with-options`
## Getting a stacktrace
The best start to investigate a crash such as an segmentation fault is a backtrace.
You can set `RUST_BACKTRACE=1` as an environment variable to get the stack trace on a `panic!`.
Alternatively you can use a debugger such as `gdb` to explore the issue.
Rust provides a wrapper, `rust-gdb`, which has pretty-printers for inspecting Rust variables.
Since PyO3 uses `cdylib` for Python shared objects, it does not receive the pretty-print debug hooks in `rust-gdb` ([rust-lang/rust#96365](https://github.com/rust-lang/rust/issues/96365)).
The mentioned issue contains a workaround for enabling pretty-printers in this case.
- Link against a debug build of python as described in the previous chapter
- Run `rust-gdb <my-binary>`
- Set a breakpoint (`b`) on `rust_panic` if you are investigating a `panic!`
- Enter `r` to run
- After the crash occurred, enter `bt` or `bt full` to print the stacktrace
Often it is helpful to run a small piece of Python code to exercise a section of Rust.
```console
rust-gdb --args python -c "import my_package; my_package.sum_to_string(1, 2)"
```
## Setting breakpoints in your Rust code
One of the preferred ways by developers to debug their code is by setting breakpoints.
This can be achieved in PyO3 by using a debugger like `rust-gdb` or `rust-lldb` with your Python interpreter.
For more information about how to use both `lldb` and `gdb` you can read the [gdb to lldb command map](https://lldb.llvm.org/use/map.html) from the lldb documentation.
### Common setup
1. Compile your extension module with debug symbols:
```bash
# Debug is the default for maturin, but you can explicitly ensure debug symbols with:
RUSTFLAGS="-g" maturin develop
# For setuptools-rust users:
pip install -e .
```
> **Note**: When using debuggers, make sure that `python` resolves to an actual Python binary or symlink and not a shim script.
> Some tools like pyenv use shim scripts which can interfere with debugging.
### Debugger specific setup
Depending on your OS and your preferences you can use two different debuggers, `rust-gdb` or `rust-lldb`.
{{#tabs }}
{{#tab name="Using rust-gdb" }}
1. Launch rust-gdb with the Python interpreter:
```bash
rust-gdb --args python
```
2. Once in gdb, set a breakpoint in your Rust code:
```bash
(gdb) break your_module.rs:42
```
3. Run your Python script that imports and uses your Rust extension module:
```bash
# Option 1: Run an inline Python command
(gdb) run -c "import your_module; your_module.your_function()"
# Option 2: Run a Python script
(gdb) run your_script.py
# Option 3: Run pytest tests
(gdb) run -m pytest tests/test_something.py::TestName
```
{{#endtab }}
{{#tab name="Using rust-lldb (for macOS users)" }}
1. Start rust-lldb with Python:
```bash
rust-lldb -- python
```
2. Set breakpoints in your Rust code:
```bash
(lldb) breakpoint set --file your_module.rs --line 42
```
3. Run your Python script:
```bash
# Option 1: Run an inline Python command
(lldb) run -c "import your_module; your_module.your_function()"
# Option 2: Run a Python script
(lldb) run your_script.py
# Option 3: Run pytest tests
(lldb) run -m pytest tests/test_something.py::TestName
```
{{#endtab }}
{{#endtabs }}
### Using VS Code
VS Code with the Rust and Python extensions provides an integrated debugging experience:
1. First, install the necessary VS Code extensions:
- [Rust Analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer)
- [CodeLLDB](https://marketplace.visualstudio.com/items?itemName=vadimcn.vscode-lldb)
- [Python](https://marketplace.visualstudio.com/items?itemName=ms-python.python)
2. Create a `.vscode/launch.json` file with a configuration that uses the LLDB Debug Launcher:
```json
{
"version": "0.2.0",
"configurations": [
{
"name": "Debug PyO3",
"type": "lldb",
"request": "attach",
"program": "${workspaceFolder}/.venv/bin/python",
"pid": "${command:pickProcess}",
"sourceLanguages": ["rust"]
},
{
"name": "Launch Python with PyO3",
"type": "lldb",
"request": "launch",
"program": "${workspaceFolder}/.venv/bin/python",
"args": ["${file}"],
"cwd": "${workspaceFolder}",
"sourceLanguages": ["rust"]
},
{
"name": "Debug PyO3 with Args",
"type": "lldb",
"request": "launch",
"program": "${workspaceFolder}/.venv/bin/python",
"args": ["path/to/your/script.py", "arg1", "arg2"],
"cwd": "${workspaceFolder}",
"sourceLanguages": ["rust"]
},
{
"name": "Debug PyO3 Tests",
"type": "lldb",
"request": "launch",
"program": "${workspaceFolder}/.venv/bin/python",
"args": ["-m", "pytest", "tests/your_test.py::test_function", "-v"],
"cwd": "${workspaceFolder}",
"sourceLanguages": ["rust"]
}
]
}
```
This configuration supports multiple debugging scenarios:
- Attaching to a running Python process
- Launching the currently open Python file
- Running a specific script with command-line arguments
- Running pytest tests
3. Set breakpoints in your Rust code by clicking in the gutter next to line numbers.
4. Start debugging:
- For attaching to a running Python process: First start the process, then select the "Debug PyO3" configuration and click Start Debugging (F5).
You'll be prompted to select the Python process to attach to.
- For launching a Python script: Open your Python script, select the "Launch Python with PyO3" configuration and click Start Debugging (F5).
- For running with arguments: Select "Debug PyO3 with Args" (remember to edit the configuration with your actual script path and arguments).
- For running tests: Select "Debug PyO3 Tests" (edit the test path as needed).
5. When debugging PyO3 code:
- You can inspect Rust variables and data structures
- Use the debug console to evaluate expressions
- Step through Rust code line by line using the step controls
- Set conditional breakpoints for more complex debugging scenarios
### Advanced Debugging Configurations
For advanced debugging scenarios, you might want to add environment variables or enable specific Rust debug flags:
```json
{
"name": "Debug PyO3 with Environment",
"type": "lldb",
"request": "launch",
"program": "${workspaceFolder}/.venv/bin/python",
"args": ["${file}"],
"env": {
"RUST_BACKTRACE": "1",
"PYTHONPATH": "${workspaceFolder}"
},
"sourceLanguages": ["rust"]
}
```
### Debugging from Jupyter Notebooks
For Jupyter Notebooks run from VS Code, you can use the following helper functions to automate the launch configuration:
```python
from pathlib import Path
import os
import json
import sys
def update_launch_json(vscode_config_file_path=None):
"""Update VSCode launch.json with the correct Jupyter kernel PID.
Args:
vscode_config_file_path (str, optional): Path to the .vscode/launch.json file.
If not provided, will use the current working directory.
"""
pid = get_jupyter_kernel_pid()
if not pid:
print("Could not determine Jupyter kernel PID.")
return
# Determine launch.json path
if vscode_config_file_path:
launch_json_path = vscode_config_file_path
else:
launch_json_path = os.path.join(Path(os.getcwd()), ".vscode", "launch.json")
# Get Python interpreter path
python_path = sys.executable
# Default debugger config
debug_config = {
"version": "0.2.0",
"configurations": [
{
"name": "Debug PyO3 (Jupyter)",
"type": "lldb",
"request": "attach",
"program": python_path,
"pid": pid,
"sourceLanguages": ["rust"],
},
{
"name": "Launch Python with PyO3",
"type": "lldb",
"request": "launch",
"program": python_path,
"args": ["${file}"],
"cwd": "${workspaceFolder}",
"sourceLanguages": ["rust"]
}
],
}
# Create .vscode directory if it doesn't exist
try:
os.makedirs(os.path.dirname(launch_json_path), exist_ok=True)
# If launch.json already exists, try to update it instead of overwriting
if os.path.exists(launch_json_path):
try:
with open(launch_json_path, "r") as f:
existing_config = json.load(f)
# Check if our configuration already exists
config_exists = False
for config in existing_config.get("configurations", []):
if config.get("name") == "Debug PyO3 (Jupyter)":
config["pid"] = pid
config["program"] = python_path
config_exists = True
if not config_exists:
existing_config.setdefault("configurations", []).append(debug_config["configurations"][0])
debug_config = existing_config
except Exception:
# If reading fails, we'll just overwrite with our new configuration
pass
with open(launch_json_path, "w") as f:
json.dump(debug_config, f, indent=4)
print(f"Updated launch.json with PID: {pid} at {launch_json_path}")
except Exception as e:
print(f"Error updating launch.json: {e}")
def get_jupyter_kernel_pid():
"""Find the process ID (PID) of the running Jupyter kernel.
Returns:
int: The process ID of the Jupyter kernel, or None if not found.
"""
# Check if we're running in a Jupyter environment
if 'ipykernel' in sys.modules:
pid = os.getpid()
print(f"Jupyter kernel PID: {pid}")
return pid
else:
print("Not running in a Jupyter environment.")
return None
```
To use these functions:
1. Run the cell containing these functions in your Jupyter notebook
2. Run `update_launch_json()` in a cell
3. In VS Code, select the "Debug PyO3 (Jupyter)" configuration and start debugging
## Thread Safety and Compiler Sanitizers
PyO3 attempts to match the Rust language-level guarantees for thread safety, but that does not preclude other code outside of the control of PyO3 or buggy code managed by a PyO3 extension module from creating a thread safety issue.
Analyzing whether or not a piece of Rust code that uses the CPython C API is thread safe can be quite complicated, since many Python operations can lead to arbitrary Python code execution.
Automated ways to discover thread safety issues can often be more fruitful than code analysis.
[ThreadSanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html) is a thread safety checking runtime that can be used to detect data races triggered by thread safety bugs or incorrect use of thread-unsafe data structures.
While it can only detect data races triggered by code at runtime, if it does detect something the reports often point to exactly where the problem is happening.
To use `ThreadSanitizer` with a library that depends on PyO3, you will need to install a nightly Rust toolchain, along with the `rust-src` component, since you will need to compile the Rust standard library:
```bash
rustup install nightly
rustup override set nightly
rustup component add rust-src
```
You will also need a version of CPython compiled using LLVM/Clang with the same major version of LLVM as is currently used to compile nightly Rust.
As of March 2025, Rust nightly uses LLVM 20.
The [cpython_sanity docker images](https://github.com/nascheme/cpython_sanity) contain a development environment with a pre-compiled version of CPython 3.13 or 3.14 as well as optionally NumPy and SciPy, all compiled using LLVM 20 and ThreadSanitizer.
After activating a nightly Rust toolchain, you can build your project using `ThreadSanitizer` with the following command:
```bash
RUSTFLAGS="-Zsanitizer=thread" maturin develop -Zbuild-std --target x86_64-unknown-linux-gnu
```
If you are not running on an x86_64 Linux machine, you should replace `x86_64-unknown-linux-gnu` with the [target triple](https://doc.rust-lang.org/rustc/platform-support.html#tier-1-with-host-tools) that is appropriate for your system.
You can also replace `maturin develop` with `cargo test` to run `cargo` tests.
Note that `cargo` runs tests in a thread pool, so `cargo` tests can be a good way to find thread safety issues.
You can also replace `-Zsanitizer=thread` with `-Zsanitizer=address` or any of the other sanitizers that are [supported by Rust](https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html).
Note that you'll need to build CPython from source with the appropriate [configure script flags](https://docs.python.org/3/using/configure.html#cmdoption-with-address-sanitizer) to use the same sanitizer environment as you want to use for your Rust code.
+5
View File
@@ -0,0 +1,5 @@
# The PyO3 ecosystem
This portion of the guide is dedicated to crates which are external to the main PyO3 project and provide additional functionality you might find useful.
Because these projects evolve independently of the PyO3 repository the content of these articles may fall out of date over time; please file issues on the PyO3 GitHub to alert maintainers when this is the case.
+15
View File
@@ -0,0 +1,15 @@
# Using `async` and `await`
*`async`/`await` support is currently being integrated in PyO3.*
*See the [dedicated documentation](../async-await.md)*
If you are working with a Python library that makes use of async functions or wish to provide Python bindings for an async Rust library, [`pyo3-async-runtimes`](https://github.com/PyO3/pyo3-async-runtimes) likely has the tools you need.
It provides conversions between async functions in both Python and Rust and was designed with first-class support for popular Rust runtimes such as [`tokio`](https://tokio.rs/) and [`async-std`](https://docs.rs/async-std/latest/async_std/).
In addition, all async Python code runs on the default `asyncio` event loop, so `pyo3-async-runtimes` should work just fine with existing Python libraries.
## Additional Information
- Managing event loop references can be tricky with `pyo3-async-runtimes`.
See [Event Loop References](https://docs.rs/pyo3-async-runtimes/#event-loop-references-and-contextvars) in the API docs to get a better intuition for how event loop references are managed in this library.
- Testing `pyo3-async-runtimes` libraries and applications requires a custom test harness since Python requires control over the main thread.
You can find a testing guide in the [API docs for the `testing` module](https://docs.rs/pyo3-async-runtimes/latest/pyo3_async_runtimes/testing)
+93
View File
@@ -0,0 +1,93 @@
# Logging
It is desirable if both the Python and Rust parts of the application end up logging using the same configuration into the same place.
This section of the guide briefly discusses how to connect the two languages' logging ecosystems together.
The recommended way for Python extension modules is to configure Rust's logger to send log messages to Python using the `pyo3-log` crate.
For users who want to do the opposite and send Python log messages to Rust, see the note at the end of this guide.
## Using `pyo3-log` to send Rust log messages to Python
The [pyo3-log] crate allows sending the messages from the Rust side to Python's [logging] system.
This is mostly suitable for writing extension modules to consume from Python programs.
Use [`pyo3_log::init`][init] to install the logger in its default configuration.
It's also possible to tweak its configuration (mostly to tune its performance).
```rust,no_run
#[pyo3::pymodule]
mod my_module {
use log::info;
use pyo3::prelude::*;
#[pyfunction]
fn log_something() {
// This will use the logger installed in `my_module` to send the `info`
// message to the Python logging facilities.
info!("Something!");
}
#[pymodule_init]
fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
// A good place to install the Rust -> Python logger.
pyo3_log::init();
}
}
```
Then it is up to the Python side to actually output the messages somewhere.
```python
import logging
import my_module
FORMAT = '%(levelname)s %(name)s %(asctime)-15s %(filename)s:%(lineno)d %(message)s'
logging.basicConfig(format=FORMAT)
logging.getLogger().setLevel(logging.INFO)
my_module.log_something()
```
It is important to initialize the Python loggers first, before calling any Rust functions that may log.
This limitation can be worked around if it is not possible to satisfy, read the documentation about [caching].
## The Python to Rust direction
To have python logs be handled by Rust, one need only register a rust function to handle logs emitted from the core python logging module.
This has been implemented within the [pyo3-pylogger] crate.
```rust,no_run
use log::{info, warn};
use pyo3::prelude::*;
fn main() -> PyResult<()> {
// register the host handler with python logger, providing a logger target
// set the name here to something appropriate for your application
pyo3_pylogger::register("example_application_py_logger");
// initialize up a logger
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("trace")).init();
// Log some messages from Rust.
info!("Just some normal information!");
warn!("Something spooky happened!");
// Log some messages from Python
Python::attach(|py| {
py.run(
"
import logging
logging.error('Something bad happened')
",
None,
None,
)
})
}
```
[logging]: https://docs.python.org/3/library/logging.html
[pyo3-log]: https://crates.io/crates/pyo3-log
[init]: https://docs.rs/pyo3-log/*/pyo3_log/fn.init.html
[caching]: https://docs.rs/pyo3-log/*/pyo3_log/#performance-filtering-and-caching
[pyo3-pylogger]: https://crates.io/crates/pyo3-pylogger
+96
View File
@@ -0,0 +1,96 @@
# Tracing
Python projects that write extension modules for performance reasons may want to tap into [Rust's `tracing` ecosystem] to gain insight into the performance of their extension module.
This section of the guide describes a few crates that provide ways to do that.
They build on [`tracing_subscriber`][tracing-subscriber] and require code changes in both Python and Rust to integrate.
Note that each extension module must configure its own `tracing` integration; one extension module will not see `tracing` data from a different module.
## `pyo3-tracing-subscriber` ([documentation][pyo3-tracing-subscriber-docs])
[`pyo3-tracing-subscriber`][pyo3-tracing-subscriber] provides a way for Python projects to configure `tracing_subscriber`.
It exposes a few `tracing_subscriber` layers:
- `tracing_subscriber::fmt` for writing human-readable output to file or stdout
- `opentelemetry-stdout` for writing OTLP output to file or stdout
- `opentelemetry-otlp` for writing OTLP output to an OTLP endpoint
The extension module must call [`pyo3_tracing_subscriber::add_submodule`][add-submodule] to export the Python classes needed to configure and initialize `tracing`.
On the Python side, use the `Tracing` context manager to initialize tracing and run Rust code inside the context manager's block. `Tracing` takes a `GlobalTracingConfig` instance describing the layers to be used.
See [the README on crates.io][pyo3-tracing-subscriber] for example code.
## `pyo3-python-tracing-subscriber` ([documentation][pyo3-python-tracing-subscriber-docs])
The similarly-named [`pyo3-python-tracing-subscriber`][pyo3-python-tracing-subscriber] implements a shim in Rust that forwards `tracing` data to a `Layer` implementation defined in and passed in from Python.
There are many ways an extension module could integrate `pyo3-python-tracing-subscriber` but a simple one may look something like this:
```rust,no_run
#[tracing::instrument]
#[pyfunction]
fn fibonacci(index: usize, use_memoized: bool) -> PyResult<usize> {
// ...
}
#[pyfunction]
pub fn initialize_tracing(py_impl: Bound<'_, PyAny>) {
tracing_subscriber::registry()
.with(pyo3_python_tracing_subscriber::PythonCallbackLayerBridge::new(py_impl))
.init();
}
```
The extension module must provide some way for Python to pass in one or more Python objects that implement [the `Layer` interface].
Then it should construct [`pyo3_python_tracing_subscriber::PythonCallbackLayerBridge`][PythonCallbackLayerBridge] instances with each of those Python objects and initialize `tracing_subscriber` as shown above.
The Python objects implement a modified version of the `Layer` interface:
- `on_new_span()` may return some state that will stored inside the Rust span
- other callbacks will be given that state as an additional positional argument
A dummy `Layer` implementation may look like this:
```python
import rust_extension
class MyPythonLayer:
def __init__(self):
pass
# `on_new_span` can return some state
def on_new_span(self, span_attrs: str, span_id: str) -> int:
print(f"[on_new_span]: {span_attrs} | {span_id}")
return random.randint(1, 1000)
# The state from `on_new_span` is passed back into other trait methods
def on_event(self, event: str, state: int):
print(f"[on_event]: {event} | {state}")
def on_close(self, span_id: str, state: int):
print(f"[on_close]: {span_id} | {state}")
def on_record(self, span_id: str, values: str, state: int):
print(f"[on_record]: {span_id} | {values} | {state}")
def main():
rust_extension.initialize_tracing(MyPythonLayer())
print("10th fibonacci number: ", rust_extension.fibonacci(10, True))
```
`pyo3-python-tracing-subscriber` has [working examples] showing both the Rust side and the Python side of an integration.
[pyo3-tracing-subscriber]: https://crates.io/crates/pyo3-tracing-subscriber
[pyo3-tracing-subscriber-docs]: https://docs.rs/pyo3-tracing-subscriber
[add-submodule]: https://docs.rs/pyo3-tracing-subscriber/*/pyo3_tracing_subscriber/fn.add_submodule.html
[pyo3-python-tracing-subscriber]: https://crates.io/crates/pyo3-python-tracing-subscriber
[pyo3-python-tracing-subscriber-docs]: https://docs.rs/pyo3-python-tracing-subscriber
[PythonCallbackLayerBridge]: https://docs.rs/pyo3-python-tracing-subscriber/*/pyo3_python_tracing_subscriber/struct.PythonCallbackLayerBridge.html
[working examples]: https://github.com/getsentry/pyo3-python-tracing-subscriber/tree/main/demo
[Rust's `tracing` ecosystem]: https://crates.io/crates/tracing
[tracing-subscriber]: https://docs.rs/tracing-subscriber/*/tracing_subscriber/
[the `Layer` interface]: https://docs.rs/tracing-subscriber/*/tracing_subscriber/layer/trait.Layer.html
+180
View File
@@ -0,0 +1,180 @@
# Python exceptions
## Defining a new exception
Use the [`create_exception!`] macro:
```rust
use pyo3::create_exception;
create_exception!(module, MyError, pyo3::exceptions::PyException);
```
- `module` is the name of the containing module.
- `MyError` is the name of the new exception type.
For example:
```rust
use pyo3::prelude::*;
use pyo3::create_exception;
use pyo3::types::IntoPyDict;
use pyo3::exceptions::PyException;
create_exception!(mymodule, CustomError, PyException);
# fn main() -> PyResult<()> {
Python::attach(|py| {
let ctx = [("CustomError", py.get_type::<CustomError>())].into_py_dict(py)?;
pyo3::py_run!(
py,
*ctx,
"assert str(CustomError) == \"<class 'mymodule.CustomError'>\""
);
pyo3::py_run!(py, *ctx, "assert CustomError('oops').args == ('oops',)");
# Ok(())
})
# }
```
When using PyO3 to create an extension module, you can add the new exception to the module like this, so that it is importable from Python:
```rust,no_run
# fn main() {}
use pyo3::prelude::*;
use pyo3::exceptions::PyException;
pyo3::create_exception!(mymodule, CustomError, PyException);
#[pymodule]
mod mymodule {
#[pymodule_export]
use super::CustomError;
// ... other elements added to module ...
}
```
## Raising an exception
As described in the [function error handling](./function/error-handling.md) chapter, to raise an exception from a `#[pyfunction]` or `#[pymethods]`, return an `Err(PyErr)`.
PyO3 will automatically raise this exception for you when returning the result to Python.
You can also manually write and fetch errors in the Python interpreter's global state:
```rust
use pyo3::{Python, PyErr};
use pyo3::exceptions::PyTypeError;
Python::attach(|py| {
PyTypeError::new_err("Error").restore(py);
assert!(PyErr::occurred(py));
drop(PyErr::fetch(py));
});
```
## Checking exception types
Python has an [`isinstance`](https://docs.python.org/3/library/functions.html#isinstance) method to check an object's type.
In PyO3 every object has the [`PyAny::is_instance`] and [`PyAny::is_instance_of`] methods which do the same thing.
```rust,no_run
use pyo3::prelude::*;
use pyo3::types::{PyBool, PyList};
# fn main() -> PyResult<()> {
Python::attach(|py| {
assert!(PyBool::new(py, true).is_instance_of::<PyBool>());
let list = PyList::new(py, &[1, 2, 3, 4])?;
assert!(!list.is_instance_of::<PyBool>());
assert!(list.is_instance_of::<PyList>());
# Ok(())
})
# }
```
To check the type of an exception, you can similarly do:
```rust,no_run
# use pyo3::exceptions::PyTypeError;
# use pyo3::prelude::*;
# Python::attach(|py| {
# let err = PyTypeError::new_err(());
err.is_instance_of::<PyTypeError>(py);
# });
```
## Using exceptions defined in Python code
It is possible to use an exception defined in Python code as a native Rust type.
The [`import_exception!`] macro allows importing a specific exception class and defines a Rust type for that exception.
```rust,no_run
#![allow(dead_code)]
use pyo3::prelude::*;
mod io {
pyo3::import_exception!(io, UnsupportedOperation);
}
fn tell(file: &Bound<'_, PyAny>) -> PyResult<u64> {
match file.call_method0("tell") {
Err(_) => Err(io::UnsupportedOperation::new_err("not supported: tell")),
Ok(x) => x.extract::<u64>(),
}
}
```
[`pyo3::exceptions`]({{#PYO3_DOCS_URL}}/pyo3/exceptions/index.html) defines exceptions for several standard library modules.
## Creating more complex exceptions
If you need to create an exception with more complex behavior, you can also manually create a subclass of `PyException`:
```rust
#![allow(dead_code)]
# #[cfg(any(not(Py_LIMITED_API), Py_3_12))] {
use pyo3::prelude::*;
use pyo3::types::IntoPyDict;
use pyo3::exceptions::PyException;
#[pyclass(extends=PyException)]
struct CustomError {
#[pyo3(get)]
url: String,
#[pyo3(get)]
message: String,
}
#[pymethods]
impl CustomError {
#[new]
fn new(url: String, message: String) -> Self {
Self { url, message }
}
}
# fn main() -> PyResult<()> {
Python::attach(|py| {
let ctx = [("CustomError", py.get_type::<CustomError>())].into_py_dict(py)?;
pyo3::py_run!(
py,
*ctx,
"assert str(CustomError) == \"<class 'builtins.CustomError'>\", repr(CustomError)"
);
pyo3::py_run!(py, *ctx, "assert CustomError('https://example.com', 'something went bad').args == ('https://example.com', 'something went bad')");
pyo3::py_run!(py, *ctx, "assert CustomError('https://example.com', 'something went bad').url == 'https://example.com'");
# Ok(())
})
# }
# }
```
Note that when the `abi3` feature is enabled, subclassing `PyException` is only possible on Python 3.12 or greater.
[`create_exception!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.create_exception.html
[`import_exception!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.import_exception.html
[`PyAny::is_instance`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.is_instance
[`PyAny::is_instance_of`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.is_instance_of
+199
View File
@@ -0,0 +1,199 @@
# Frequently Asked Questions and troubleshooting
Sorry that you're having trouble using PyO3.
If you can't find the answer to your problem in the list below, you can also reach out for help on [GitHub Discussions](https://github.com/PyO3/pyo3/discussions) and on [Discord](https://discord.gg/33kcChzH7f).
## I'm experiencing deadlocks using PyO3 with `std::sync::OnceLock`, `std::sync::LazyLock`, `lazy_static`, and `once_cell`
`OnceLock`, `LazyLock`, and their thirdparty predecessors use blocking to ensure only one thread ever initializes them.
Because the Python interpreter can introduce additional locks (the Python GIL and GC can both require all other threads to pause) this can lead to deadlocks in the following way:
1. A thread (thread A) which is attached to the Python interpreter starts initialization of a `OnceLock` value.
2. The initialization code calls some Python API which temporarily detaches from the interpreter e.g. `Python::import`.
3. Another thread (thread B) attaches to the Python interpreter and attempts to access the same `OnceLock` value.
4. Thread B is blocked, because it waits for `OnceLock`'s initialization to lock to release.
5. On GIL-enabled Python, thread A is now also blocked, because it waits to re-attach to the interpreter (by taking the GIL which thread B still holds).
6. Deadlock.
PyO3 provides a struct [`PyOnceLock`] which implements a single-initialization API based on these types that avoids deadlocks.
You can also make use of the [`OnceExt`] and [`OnceLockExt`] extension traits that enable using the standard library types for this purpose by providing new methods for these types that avoid the risk of deadlocking with the Python interpreter.
This means they can be used in place of other choices when you are experiencing the deadlock described above.
See the documentation for [`PyOnceLock`] and [`OnceExt`] for further details and an example how to use them.
[`PyOnceLock`]: {{#PYO3_DOCS_URL}}/pyo3/sync/struct.PyOnceLock.html
[`OnceExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html
[`OnceLockExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceLockExt.html
## I can't run `cargo test`; or I can't build in a Cargo workspace: I'm having linker issues like "Symbol not found" or "Undefined reference to _PyExc_SystemError"
The `extension-module` feature (now deprecated) disables linking to `libpython`.
This breaks binaries and tests which need to load symbols from `libpython` to execute.
Remove the `extension-module` feature and upgrade to `maturin >= 1.9.4` or `setuptools-rust 1.12`.
If building manually, see the [`PYO3_BUILD_EXTENSION_MODULE` environment variable](./building-and-distribution.md#the-pyo3_build_extension_module-environment-variable).
## I can't run `cargo test`: my crate cannot be found for tests in `tests/` directory
The Rust book suggests to [put integration tests inside a `tests/` directory](https://doc.rust-lang.org/book/ch11-03-test-organization.html#integration-tests).
For a PyO3 project where the `crate-type` is set to `"cdylib"` in your `Cargo.toml`, the compiler won't be able to find your crate and will display errors such as `E0432` or `E0463`:
```text
error[E0432]: unresolved import `my_crate`
--> tests/test_my_crate.rs:1:5
|
1 | use my_crate;
| ^^^^^^^^^^^^ no external crate `my_crate`
```
The best solution is to make your crate types include both `rlib` and `cdylib`:
```toml
# Cargo.toml
[lib]
crate-type = ["cdylib", "rlib"]
```
## Ctrl-C doesn't do anything while my Rust code is executing
This is because Ctrl-C raises a SIGINT signal, which is handled by the calling Python process by simply setting a flag to action upon later.
This flag isn't checked while Rust code called from Python is executing, only once control returns to the Python interpreter.
You can give the Python interpreter a chance to process the signal properly by calling `Python::check_signals`.
It's good practice to call this function regularly if you have a long-running Rust function so that your users can cancel it.
## `#[pyo3(get)]` clones my field
You may have a nested struct similar to this:
```rust,no_run
# use pyo3::prelude::*;
#[pyclass(from_py_object)]
#[derive(Clone)]
struct Inner {/* fields omitted */}
#[pyclass]
struct Outer {
#[pyo3(get)]
inner: Inner,
}
#[pymethods]
impl Outer {
#[new]
fn __new__() -> Self {
Self { inner: Inner {} }
}
}
```
When Python code accesses `Outer`'s field, PyO3 will return a new object on every access (note that their addresses are different):
```python
outer = Outer()
a = outer.inner
b = outer.inner
assert a is b, f"a: {a}\nb: {b}"
```
```text
AssertionError: a: <builtins.Inner object at 0x00000238FFB9C7B0>
b: <builtins.Inner object at 0x00000238FFB9C830>
```
This can be especially confusing if the field is mutable, as getting the field and then mutating it won't persist - you'll just get a fresh clone of the original on the next access.
Unfortunately Python and Rust don't agree about ownership - if PyO3 gave out references to (possibly) temporary Rust objects to Python code, Python code could then keep that reference alive indefinitely.
Therefore returning Rust objects requires cloning.
If you don't want that cloning to happen, a workaround is to allocate the field on the Python heap and store a reference to that, by using [`Py<...>`]({{#PYO3_DOCS_URL}}/pyo3/struct.Py.html):
```rust,no_run
# use pyo3::prelude::*;
#[pyclass]
struct Inner {/* fields omitted */}
#[pyclass]
struct Outer {
inner: Py<Inner>,
}
#[pymethods]
impl Outer {
#[new]
fn __new__(py: Python<'_>) -> PyResult<Self> {
Ok(Self {
inner: Py::new(py, Inner {})?,
})
}
#[getter]
fn inner(&self, py: Python<'_>) -> Py<Inner> {
self.inner.clone_ref(py)
}
}
```
This time `a` and `b` *are* the same object:
```python
outer = Outer()
a = outer.inner
b = outer.inner
assert a is b, f"a: {a}\nb: {b}"
print(f"a: {a}\nb: {b}")
```
```text
a: <builtins.Inner object at 0x0000020044FCC670>
b: <builtins.Inner object at 0x0000020044FCC670>
```
The downside to this approach is that any Rust code working on the `Outer` struct potentially has to attach to the Python interpreter to do anything with the `inner` field. (If `Inner` is `#[pyclass(frozen)]` and implements `Sync`, then `Py::get` may be used to access the `Inner` contents from `Py<Inner>` without needing to attach to the interpreter.)
## I want to use the `pyo3` crate re-exported from dependency but the proc-macros fail
All PyO3 proc-macros (`#[pyclass]`, `#[pyfunction]`, `#[derive(FromPyObject)]` and so on) expect the `pyo3` crate to be available under that name in your crate root, which is the normal situation when `pyo3` is a direct dependency of your crate.
However, when the dependency is renamed, or your crate only indirectly depends on `pyo3`, you need to let the macro code know where to find the crate.
This is done with the `crate` attribute:
```rust,no_run
# use pyo3::prelude::*;
# pub extern crate pyo3;
# mod reexported { pub use ::pyo3; }
# #[allow(dead_code)]
#[pyclass]
#[pyo3(crate = "reexported::pyo3")]
struct MyClass;
```
## I'm trying to call Python from Rust but I get `STATUS_DLL_NOT_FOUND` or `STATUS_ENTRYPOINT_NOT_FOUND`
This happens on Windows when linking to the python DLL fails or the wrong one is linked.
The Python DLL on Windows will usually be called something like:
- `python3X.dll` for Python 3.X, e.g. `python310.dll` for Python 3.10
- `python3.dll` when using PyO3's `abi3` feature
- `python3t.dll` when using PyO3's `abi3t` feature
The DLL needs to be locatable using the [Windows DLL search order](https://learn.microsoft.com/en-us/windows/win32/dlls/dynamic-link-library-search-order#standard-search-order-for-unpackaged-apps).
Some ways to achieve this are:
- Put the Python DLL in the same folder as your build artifacts
- Add the directory containing the Python DLL to your `PATH` environment variable, for example `C:\Users\<You>\AppData\Local\Programs\Python\Python310`
- If this happens when you are *distributing* your program, consider using [PyOxidizer](https://github.com/indygreg/PyOxidizer) to package it with your binary.
- The legacy MSI Python installer for Python 3.15 installs `python3t.dll` into an `abi3t_compat` subfolder of the main Python installation.
Add that folder to your PATH as well if you are using `abi3t` features on Python 3.15.
If the wrong DLL is linked it is possible that this happened because another program added itself and its own Python DLLs to `PATH`.
Rearrange your `PATH` variables to give the correct DLL priority.
> [!NOTE]
> Changes to `PATH` (or any other environment variable) are not visible to existing shells. Restart it for changes to take effect.
For advanced troubleshooting, [Dependency Walker](https://www.dependencywalker.com/) can be used to diagnose linking errors.
+325
View File
@@ -0,0 +1,325 @@
# Features reference
PyO3 provides a number of Cargo features to customize functionality.
This chapter of the guide provides detail on each of them.
By default, only the `macros` feature is enabled.
## Features to select the target ABI
Each version of CPython exposes an ABI corresponding to the C API that is particular to that version.
Extensions built for this "version-specific" ABI can only be imported by Python interpreters with the same major version, since the Python C API and ABI are subject to change in every major release.
Python also supports a [limited API](https://docs.python.org/3/c-api/stable.html#limited-c-api) and [stable ABI](https://docs.python.org/3/c-api/stable.html#stable-abi).
Extensions targeting the stable ABI defined by Python 3.X can be imported by any subsequent Python version newer than Python 3.X, but not older versions.
There are two "flavors" of stable ABI: `abi3`, supported on Python 3.2 and newer but *not* the free-threaded builds and `abi3t`, supported on Python 3.15 and newer for both the GIL-enabled and free-threaded builds of Python.
PyO3 supports building extensions targeting both flavors of stable ABI.
When PyO3 finds a "host" python interpreter and no `abi3` or `abi3t` feature is active, it will generate extensions targeting the version-specific ABI for the host Python version.
For example, when ``pip`` or ``uv`` installs a Python package that includes Rust dependencies that depend on PyO3, the host interpreter is the interpreter running ``pip`` or the interpreter from the activated ``uv`` environment.
To build for the stable ABIs, you must activate an `abi3` and/or an `abi3t` feature.
You can simultaneously activate both features.
This will produce an `abi3` extension on the GIL-enabled and a version-specific extension on Python 3.14 and an `abi3t` extension on Python 3.15 and newer.
### `abi3`
This feature is used when building Python extension modules to create wheels which are compatible with multiple Python versions but not the free-threaded build.
It restricts PyO3's API to a subset of the full Python API which is guaranteed by [PEP 384](https://www.python.org/dev/peps/pep-0384/) to be forwards-compatible with future Python versions.
Since no particular "target" Python version is implied by this feature, it will generate an `abi3` extension targeting the stable ABI defined by the "host" Python interpreter.
As such, this feature requires a host Python interpreter is present and cannot be used for cross-compilation.
If you are using this feature to distribute extensions, you should ensure your minimum supported Python version is installed to do the build.
However, see the description of the `abi3-pyXY` feature if you would like to generate extensions targeting an older Python version using a recent Python interpreter.
See the [building and distribution](building-and-distribution.md#py_limited_apiabi3abi3t) section for further detail.
### The `abi3-pyXY` features
(`abi3-py38`, `abi3-py39`, `abi3-py310`, `abi3-py311`, `abi3-py312`, `abi3-py313` and `abi3-py314`)
These features are extensions of the `abi3` feature to specify the exact minimum Python version which the multiple-version-wheel will support.
They can be used to target versions of the ABI defined by Python versions released before the "host" python interpreter.
They can also be used without any host interpreter at all to cross-compile.
See the [building and distribution](building-and-distribution.md#minimum-python-version-for-abi3-and-abi3t-builds) section for further detail.
### `abi3t`
This feature is used when building Python extension modules to create wheels which are compatible with Python 3.15 or newer, targeting both the free-threaded and GIL-enabled build with a single extension module.
It restricts PyO3's API to a subset of the full Python API which is guaranteed by [PEP 384](https://www.python.org/dev/peps/pep-0384/) and [PEP 803](https://www.python.org/dev/peps/pep-0803/) to be forwards-compatible with future Python versions and be importable by the free-threaded and GIL-enabled interpreter.
Since no particular "target" Python version is implied by this feature, it will generate an `abi3` extension targeting the stable ABI defined by the "host" Python interpreter.
As such, this feature requires a host Python interpreter is present and cannot be used for cross-compilation.
If you are using this feature to distribute extensions, you should ensure your minimum supported Python version is installed to do the build.
However, see the description of the `abi3t-pyXY` feature if you would like to generate extensions targeting an older Python version using a recent Python interpreter.
Activating an `abi3t` and targeting Python 3.14 and older is a no-op: the build will produce an extension but it will not be an `abi3t` extension, since `abi3t` is unsupported before Python 3.15.
See the [building and distribution](building-and-distribution.md#py_limited_apiabi3abi3t) section for further detail.
### The `abi3t-pyXY` features
(`abi3t-py315`)
These features are extensions of the `abi3t` feature to specify the exact minimum Python version which the multiple-version-wheel will support.
They can be used to target versions of the ABI defined by Python versions released before the "host" python interpreter.
They can also be used without any host interpreter at all to cross-compile.
Activating an `abi3t` and targeting Python 3.14 and older is a no-op: the build will produce an extension but it will not be an `abi3t` extension, since `abi3t` is unsupported before Python 3.15.
See the [building and distribution](building-and-distribution.md#minimum-python-version-for-abi3-and-abi3t-builds) section for further detail.
## Features for embedding Python in Rust
### `auto-initialize`
This feature changes [`Python::attach`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.attach) to automatically initialize a Python interpreter (by calling [`Python::initialize`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.initialize)) if needed.
If you do not enable this feature, you should call `Python::initialize()` before attempting to call any other Python APIs.
## Advanced Features
### `experimental-async`
This feature adds support for `async fn` in `#[pyfunction]` and `#[pymethods]`.
The feature has some unfinished refinements and performance improvements.
To help finish this off, see [issue #1632](https://github.com/PyO3/pyo3/issues/1632) and its associated draft PRs.
### `experimental-inspect`
This feature adds to the built binaries introspection data that can be then retrieved using the `pyo3-introspection` crate to generate [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html).
Also, this feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types.
This is a first step towards adding first-class support for generating type annotations automatically in PyO3, however work is needed to finish this off.
All feedback and offers of help welcome on [issue #2454](https://github.com/PyO3/pyo3/issues/2454).
### `py-clone`
This feature was introduced to ease migration.
It was found that delayed reference counting (which PyO3 used historically) could not be made sound and hence `Clone`-ing an instance of `Py<T>` is impossible when not attached to Python interpreter (it will panic).
To avoid migrations introducing new panics without warning, the `Clone` implementation itself is now gated behind this feature.
### `pyo3_disable_reference_pool`
This is a performance-oriented conditional compilation flag, e.g. [set via `$RUSTFLAGS`][set-configuration-options], which disabled the global reference pool and the associated overhead for the crossing the Python-Rust boundary.
However, if enabled, `Drop`ping an instance of `Py<T>` when not attached to the Python interpreter will abort the process.
### `macros`
This feature enables a dependency on the `pyo3-macros` crate, which provides the procedural macros portion of PyO3's API:
- `#[pymodule]`
- `#[pyfunction]`
- `#[pyclass]`
- `#[pymethods]`
- `#[derive(FromPyObject)]`
It also provides the `py_run!` macro.
These macros require a number of dependencies which may not be needed by users who just need PyO3 for Python FFI.
Disabling this feature enables faster builds for those users, as these dependencies will not be built if this feature is disabled.
> [!NOTE]
> This feature is enabled by default.
> To disable it, set `default-features = false` for the `pyo3` entry in your Cargo.toml.
### `multiple-pymethods`
This feature enables each `#[pyclass]` to have more than one `#[pymethods]` block.
Most users should only need a single `#[pymethods]` per `#[pyclass]`.
In addition, not all platforms (e.g. Wasm) are supported by `inventory`, which is used in the implementation of the feature.
For this reason this feature is not enabled by default, meaning fewer dependencies and faster compilation for the majority of users.
See [the `#[pyclass]` implementation details](class.md#implementation-details) for more information.
### `nightly`
The `nightly` feature needs the nightly Rust compiler.
This allows PyO3 to use the `auto_traits` and `negative_impls` features to fix the `Python::detach` function.
## Optional Dependencies
These features enable conversions between Python types and types from other Rust crates, enabling easy access to the rest of the Rust ecosystem.
### `anyhow`
Adds a dependency on [anyhow](https://docs.rs/anyhow).
Enables a conversion from [anyhow](https://docs.rs/anyhow)’s [`Error`](https://docs.rs/anyhow/latest/anyhow/struct.Error.html) type to [`PyErr`]({{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html), for easy error handling.
### `arc_lock`
Enables Pyo3's `MutexExt` trait for all Mutexes that extend on [`lock_api::Mutex`](https://docs.rs/lock_api/latest/lock_api/struct.Mutex.html) or [`parking_lot::ReentrantMutex`](https://docs.rs/lock_api/latest/lock_api/struct.ReentrantMutex.html) and are wrapped in an [`Arc`](https://doc.rust-lang.org/std/sync/struct.Arc.html) type.
Like [`Arc<parking_lot::Mutex>`](https://docs.rs/parking_lot/latest/parking_lot/type.Mutex.html#method.lock_arc)
### `bigdecimal`
Adds a dependency on [bigdecimal](https://docs.rs/bigdecimal) and enables conversions into its [`BigDecimal`](https://docs.rs/bigdecimal/latest/bigdecimal/struct.BigDecimal.html) type.
### `bytes`
Adds a dependency on [bytes](https://docs.rs/bytes/latest/bytes) and enables conversions into its [`Bytes`](https://docs.rs/bytes/latest/bytes/struct.Bytes.html) type.
### `chrono`
Adds a dependency on [chrono](https://docs.rs/chrono).
Enables a conversion from [chrono](https://docs.rs/chrono)'s types to python:
- [TimeDelta](https://docs.rs/chrono/latest/chrono/struct.TimeDelta.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html)
- [FixedOffset](https://docs.rs/chrono/latest/chrono/offset/struct.FixedOffset.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html)
- [Utc](https://docs.rs/chrono/latest/chrono/offset/struct.Utc.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html)
- [NaiveDate](https://docs.rs/chrono/latest/chrono/naive/struct.NaiveDate.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html)
- [NaiveTime](https://docs.rs/chrono/latest/chrono/naive/struct.NaiveTime.html) -> [`PyTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTime.html)
- [DateTime](https://docs.rs/chrono/latest/chrono/struct.DateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html)
### `chrono-local`
Enables conversion from and to [Local](https://docs.rs/chrono/latest/chrono/struct.Local.html) timezones.
The current system timezone as determined by [`iana_time_zone::get_timezone()`](https://docs.rs/iana-time-zone/latest/iana_time_zone/fn.get_timezone.html) will be used for conversions.
`chrono::DateTime<Local>` will convert from either of:
- `datetime` objects with `tzinfo` equivalent to the current system timezone.
- "naive" `datetime` objects (those without a `tzinfo`), as it is a convention that naive datetime objects should be treated as using the system timezone.
When converting to Python, `Local` tzinfo is converted to a `zoneinfo.ZoneInfo` matching the current system timezone.
### `chrono-tz`
Adds a dependency on [chrono-tz](https://docs.rs/chrono-tz).
Enables conversion from and to [`Tz`](https://docs.rs/chrono-tz/latest/chrono_tz/enum.Tz.html).
### `either`
Adds a dependency on [either](https://docs.rs/either).
Enables a conversions into [either](https://docs.rs/either)’s [`Either`](https://docs.rs/either/latest/either/enum.Either.html) type.
### `eyre`
Adds a dependency on [eyre](https://docs.rs/eyre).
Enables a conversion from [eyre](https://docs.rs/eyre)’s [`Report`](https://docs.rs/eyre/latest/eyre/struct.Report.html) type to [`PyErr`]({{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html), for easy error handling.
### `hashbrown`
Adds a dependency on [hashbrown](https://docs.rs/hashbrown) and enables conversions into its [`HashMap`](https://docs.rs/hashbrown/latest/hashbrown/struct.HashMap.html) and [`HashSet`](https://docs.rs/hashbrown/latest/hashbrown/struct.HashSet.html) types.
### `indexmap`
Adds a dependency on [indexmap](https://docs.rs/indexmap) and enables conversions into its [`IndexMap`](https://docs.rs/indexmap/latest/indexmap/map/struct.IndexMap.html) type.
### `jiff-02`
Adds a dependency on [jiff@0.2](https://docs.rs/jiff/0.2).
Enables a conversion from [jiff](https://docs.rs/jiff)'s types to python:
- [SignedDuration](https://docs.rs/jiff/0.2/jiff/struct.SignedDuration.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html)
- [TimeZone](https://docs.rs/jiff/0.2/jiff/tz/struct.TimeZone.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html)
- [Offset](https://docs.rs/jiff/0.2/jiff/tz/struct.Offset.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html)
- [Date](https://docs.rs/jiff/0.2/jiff/civil/struct.Date.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html)
- [Time](https://docs.rs/jiff/0.2/jiff/civil/struct.Time.html) -> [`PyTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTime.html)
- [DateTime](https://docs.rs/jiff/0.2/jiff/civil/struct.DateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html)
- [Zoned](https://docs.rs/jiff/0.2/jiff/struct.Zoned.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html)
- [Timestamp](https://docs.rs/jiff/0.2/jiff/struct.Timestamp.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html)
- [ISOWeekDate](https://docs.rs/jiff/0.2/jiff/civil/struct.ISOWeekDate.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html)
### `lock_api`
Adds a dependency on [lock_api](https://docs.rs/lock_api) and enables Pyo3's `MutexExt` trait for all mutexes that extend on [`lock_api::Mutex`](https://docs.rs/lock_api/latest/lock_api/struct.Mutex.html) and [`parking_lot::ReentrantMutex`](https://docs.rs/lock_api/latest/lock_api/struct.ReentrantMutex.html) (like `parking_lot` or `spin`).
### `num-bigint`
Adds a dependency on [num-bigint](https://docs.rs/num-bigint) and enables conversions into its [`BigInt`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigInt.html) and [`BigUint`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigUint.html) types.
### `num-complex`
Adds a dependency on [num-complex](https://docs.rs/num-complex) and enables conversions into its [`Complex`](https://docs.rs/num-complex/latest/num_complex/struct.Complex.html) type.
### `num-rational`
Adds a dependency on [num-rational](https://docs.rs/num-rational) and enables conversions into its [`Ratio`](https://docs.rs/num-rational/latest/num_rational/struct.Ratio.html) type.
### `ordered-float`
Adds a dependency on [ordered-float](https://docs.rs/ordered-float) and enables conversions between [ordered-float](https://docs.rs/ordered-float)'s types and Python:
- [NotNan](https://docs.rs/ordered-float/latest/ordered_float/struct.NotNan.html) -> [`PyFloat`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFloat.html)
- [OrderedFloat](https://docs.rs/ordered-float/latest/ordered_float/struct.OrderedFloat.html) -> [`PyFloat`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFloat.html)
### `parking-lot`
Adds a dependency on [parking_lot](https://docs.rs/parking_lot) and enables Pyo3's `OnceExt` & `MutexExt` traits for [`parking_lot::Once`](https://docs.rs/parking_lot/latest/parking_lot/struct.Once.html) [`parking_lot::Mutex`](https://docs.rs/parking_lot/latest/parking_lot/type.Mutex.html) and [`parking_lot::ReentrantMutex`](https://docs.rs/parking_lot/latest/parking_lot/type.ReentrantMutex.html) types.
### `rust_decimal`
Adds a dependency on [rust_decimal](https://docs.rs/rust_decimal) and enables conversions into its [`Decimal`](https://docs.rs/rust_decimal/latest/rust_decimal/struct.Decimal.html) type.
### `time`
Adds a dependency on [time](https://docs.rs/time).
Enables conversions between [time](https://docs.rs/time)'s types and Python:
- [Date](https://docs.rs/time/0.3.38/time/struct.Date.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html)
- [Time](https://docs.rs/time/0.3.38/time/struct.Time.html) -> [`PyTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTime.html)
- [OffsetDateTime](https://docs.rs/time/0.3.38/time/struct.OffsetDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html)
- [PrimitiveDateTime](https://docs.rs/time/0.3.38/time/struct.PrimitiveDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html)
- [Duration](https://docs.rs/time/0.3.38/time/struct.Duration.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html)
- [UtcOffset](https://docs.rs/time/0.3.38/time/struct.UtcOffset.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html)
- [UtcDateTime](https://docs.rs/time/0.3.38/time/struct.UtcDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html)
### `serde`
Enables (de)serialization of `Py<T>` objects via [serde](https://serde.rs/).
This allows to use [`#[derive(Serialize, Deserialize)`](https://serde.rs/derive.html) on structs that hold references to `#[pyclass]` instances
```rust,no_run
# #[cfg(feature = "serde")]
# #[allow(dead_code)]
# mod serde_only {
# use pyo3::prelude::*;
# use serde::{Deserialize, Serialize};
#[pyclass]
#[derive(Serialize, Deserialize)]
struct Permission {
name: String,
}
#[pyclass]
#[derive(Serialize, Deserialize)]
struct User {
username: String,
permissions: Vec<Py<Permission>>,
}
# }
```
### `smallvec`
Adds a dependency on [smallvec](https://docs.rs/smallvec) and enables conversions into its [`SmallVec`](https://docs.rs/smallvec/latest/smallvec/struct.SmallVec.html) type.
[set-configuration-options]: https://doc.rust-lang.org/reference/conditional-compilation.html#set-configuration-options
### `uuid`
Adds a dependency on [uuid](https://docs.rs/uuid) and enables conversions into its [`Uuid`](https://docs.rs/uuid/latest/uuid/struct.Uuid.html) type.
## Deprecated features for extension module authors
### `extension-module`
Deprecated, users should remove this feature and upgrade to `maturin >= 1.9.4` or `setuptools-rust >= 1.12`.
See the [building and distribution](building-and-distribution.md#the-extension-module-feature) section for further detail.
### `generate-import-lib`
This feature is deprecated and has no effect.
+295
View File
@@ -0,0 +1,295 @@
# Supporting Free-Threaded CPython
CPython 3.14 declared support for the "free-threaded" build of CPython that does not rely on the global interpreter lock (often referred to as the GIL) for thread safety.
Since version 0.23, PyO3 supports building Rust extension modules for the free-threaded Python build and calling into free-threaded Python from Rust.
If you want more background on free-threaded Python in general, see the [what's new](https://docs.python.org/3/whatsnew/3.13.html#whatsnew313-free-threaded-cpython) entry in the 3.13 release notes (when the "free-threaded" build was first added as an experimental mode), the [free-threading HOWTO guide](https://docs.python.org/3/howto/free-threading-extensions.html#freethreading-extensions-howto) in the CPython docs, the [extension module porting guide](https://py-free-threading.github.io/porting-extensions/) in the community-maintained Python free-threading guide, and [PEP 703](https://peps.python.org/pep-0703/), which provides the technical background for the free-threading implementation in CPython.
In the GIL-enabled build (the only choice before the "free-threaded" build was introduced), the global interpreter lock serializes access to the Python runtime.
The GIL is therefore a fundamental limitation to parallel scaling of multithreaded Python workflows, due to [Amdahl's law](https://en.wikipedia.org/wiki/Amdahl%27s_law), because any time spent executing a parallel processing task on only one execution context fundamentally cannot be sped up using parallelism.
The free-threaded build removes this limit on multithreaded Python scaling.
This means it's much more straightforward to achieve parallelism using the Python [`threading`] module.
If you have ever needed to use [`multiprocessing`](https://docs.python.org/3/library/multiprocessing.html) to achieve a parallel speedup for some Python code, free-threading will likely allow the use of Python threads instead for the same workflow.
PyO3's support for free-threaded Python will enable authoring native Python extension modules that are thread-safe by construction, with much stronger safety guarantees than C extension modules.
Our goal is to enable ["fearless concurrency"](https://doc.rust-lang.org/book/ch16-00-concurrency.html) in the native Python runtime by building on the Rust [`Send` and `Sync`](https://doc.rust-lang.org/nomicon/send-and-sync.html) traits.
This document provides advice for porting Rust code using PyO3 to run under free-threaded Python.
## Supporting free-threaded Python with PyO3
Since PyO3 0.28, PyO3 defaults to assuming Python modules created with it are thread-safe.
This will be the case except for Rust code which has used `unsafe` to assume thread-safety incorrectly.
An example of this is `unsafe` code which was written with the historical assumption that Python was single-threaded due to the GIL, and so the `Python<'py>` token used by PyO3 could be used to guarantee thread-safety.
A module can opt-out of supporting free-threaded Python until it has audited its `unsafe` code for correctness by declaring the module with `#[pymodule(gil_used = true)]` (see below).
Complicated `#[pyclass]` types may need to deal with thread-safety directly; there is [a dedicated section of the guide](./class/thread-safety.md) to discuss this.
At a low-level, annotating a module sets the `Py_MOD_GIL` slot on modules defined by an extension to `Py_MOD_GIL_NOT_USED`, which allows the interpreter to see at runtime that the author of the extension module attests it is thread-safe.
By opting-out of supporting free-threaded Python, the Python interpreter will re-enable the GIL at runtime while importing your module and print a `RuntimeWarning` with a message containing the name of the module causing it to re-enable the GIL.
You can force the GIL to remain disabled by setting the `PYTHON_GIL=0` as an environment variable or passing `-Xgil=0` when starting Python (`0` means the GIL is turned off).
If you are sure that all data structures exposed in a `PyModule` are thread-safe, then pass `gil_used = false` as a parameter to the `pymodule` procedural macro declaring the module or call `PyModule::gil_used` on a `PyModule` instance.
For example:
### Example opting-in
(Note: for PyO3 versions 0.23 through 0.27, the default was `gil_used = true` and so the opposite was needed; modules needed to opt-in to free-threaded Python support with `gil_used = false`.)
```rust,no_run
/// This module relies on the GIL for thread safety
#[pyo3::pymodule(gil_used = true)]
mod my_extension {
use pyo3::prelude::*;
// this type is not thread-safe
#[pyclass]
struct MyNotThreadSafeType {
// insert not thread-safe code
}
}
```
Or for a module that is set up without using the `pymodule` macro:
```rust,no_run
use pyo3::prelude::*;
# #[allow(dead_code)]
fn register_child_module(parent_module: &Bound<'_, PyModule>) -> PyResult<()> {
let child_module = PyModule::new(parent_module.py(), "child_module")?;
child_module.gil_used(true)?;
parent_module.add_submodule(&child_module)
}
```
See the [`string-sum`](https://github.com/PyO3/pyo3/tree/main/pyo3-ffi/examples/string-sum) example for how to declare free-threaded support using raw FFI calls for modules using single-phase initialization and the [`sequential`](https://github.com/PyO3/pyo3/tree/main/pyo3-ffi/examples/sequential) example for modules using multi-phase initialization.
If you would like to use conditional compilation to trigger different code paths under the free-threaded build, you can use the `Py_GIL_DISABLED` attribute once you have configured your crate to generate the necessary build configuration data.
See [the guide section](./building-and-distribution/multiple-python-versions.md) for more details about supporting multiple different Python versions, including the free-threaded build.
## Special considerations for the free-threaded build
The free-threaded interpreter does not have a GIL.
Many existing extension modules providing mutable data structures relied on the GIL to lock Python objects and make interior mutability thread-safe.
Calling into the CPython C API is only legal when an OS thread is explicitly attached to the interpreter runtime.
In the GIL-enabled build, this happens when the GIL is acquired.
In the free-threaded build there is no GIL, but the same C macros that release or acquire the GIL in the GIL-enabled build instead ask the interpreter to attach the thread to the Python runtime, and there can be many threads simultaneously attached.
See [PEP 703](https://peps.python.org/pep-0703/#thread-states) for more background about how threads can be attached and detached from the interpreter runtime, in a manner analogous to releasing and acquiring the GIL in the GIL-enabled build.
In the GIL-enabled build, PyO3 uses the [`Python<'py>`] type and the `'py` lifetime to signify that the global interpreter lock is held.
In the freethreaded build, holding a `'py` lifetime means only that the thread is currently attached to the Python interpreter -- other threads can be simultaneously interacting with the interpreter.
### Attaching to the runtime
You still need to obtain a `'py` lifetime to interact with Python objects or call into the CPython C API.
If you are not yet attached to the Python runtime, you can register a thread using the [`Python::attach`] function.
Threads created via the Python [`threading`] module do not need to do this, and pyo3 will handle setting up the [`Python<'py>`] token when CPython calls into your extension module.
### Detaching to avoid hangs and deadlocks
The free-threaded build triggers global synchronization events in the following situations:
- During garbage collection in order to get a globally consistent view of
reference counts and references between objects
- In Python 3.13, when the first background thread is started in
order to mark certain objects as immortal
- When either `sys.settrace` or `sys.setprofile` are called in order to
instrument running code objects and threads
- During a call to `os.fork()`, to ensure a process-wide consistent state.
This is a non-exhaustive list and there may be other situations in future Python versions that can trigger global synchronization events.
This means that you should detach from the interpreter runtime using [`Python::detach`] in exactly the same situations as you should detach from the runtime in the GIL-enabled build: when doing long-running tasks that do not require the CPython runtime or when doing any task that needs to re-attach to the runtime (see the [guide section](parallelism.md#sharing-python-objects-between-rust-threads) that covers this).
In the former case, you would observe a hang on threads that are waiting on the long-running task to complete, and in the latter case you would see a deadlock while a thread tries to attach after the runtime triggers a global synchronization event, but the spawning thread prevents the synchronization event from completing.
### Exceptions and panics for multithreaded access of mutable `pyclass` instances
Data attached to `pyclass` instances is protected from concurrent access by a `RefCell`-like pattern of runtime borrow checking.
Like a `RefCell`, PyO3 will raise exceptions (or in some cases panic) to enforce exclusive access for mutable borrows.
It was always possible to generate panics like this in PyO3 in code that releases the GIL with [`Python::detach`] or calling a python method accepting `&self` from a `&mut self` (see [the docs on interior mutability](./class.md#bound-and-interior-mutability),) but now in free-threaded Python there are more opportunities to trigger these panics from Python because there is no GIL to lock concurrent access to mutably borrowed data from Python.
The most straightforward way to trigger this problem is to use the Python [`threading`] module to simultaneously call a rust function that mutably borrows a [`pyclass`]({{#PYO3_DOCS_URL}}/pyo3/attr.pyclass.html) in multiple threads.
For example, consider the following implementation:
```rust,no_run
# use pyo3::prelude::*;
#[pyclass]
#[derive(Default)]
struct ThreadIter {
count: usize,
}
#[pymethods]
impl ThreadIter {
#[new]
pub fn new() -> Self {
Default::default()
}
fn __next__(&mut self, py: Python<'_>) -> usize {
self.count += 1;
self.count
}
}
```
And then if we do something like this in Python:
```python
import concurrent.futures
from my_module import ThreadIter
i = ThreadIter()
def increment():
next(i)
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as tpe:
futures = [tpe.submit(increment) for _ in range(100)]
[f.result() for f in futures]
```
We will see an exception:
```text
Traceback (most recent call last)
File "example.py", line 5, in <module>
next(i)
RuntimeError: Already borrowed
```
We may allow user-selectable semantics for mutable pyclass definitions in a future version of PyO3, allowing some form of opt-in locking to emulate the GIL if that is needed.
For now you should explicitly add locking, possibly using conditional compilation or using the critical section API, to avoid creating deadlocks with the GIL.
### Cannot build extension modules using the limited API
The free-threaded build uses a completely new ABI and there is not yet an equivalent to the limited API for the free-threaded ABI.
That means if your crate depends on PyO3 using the `abi3` feature or an an `abi3-pyxx` feature, PyO3 will print a warning and ignore that setting when building extension modules using the free-threaded interpreter.
This means that if your package makes use of the ABI forward compatibility provided by the limited API to upload only one wheel for each release of your package, you will need to update your release procedure to also upload a version-specific free-threaded wheel.
See [the guide section](./building-and-distribution/multiple-python-versions.md) for more details about supporting multiple different Python versions, including the free-threaded build.
### Thread-safe single initialization
To initialize data exactly once, use the [`PyOnceLock`] type, which is a close equivalent to [`std::sync::OnceLock`][`OnceLock`] that also helps avoid deadlocks by detaching from the Python interpreter when threads are blocking waiting for another thread to complete initialization.
If already using [`OnceLock`] and it is impractical to replace with a [`PyOnceLock`], there is the [`OnceLockExt`] extension trait which adds [`OnceLockExt::get_or_init_py_attached`] to detach from the interpreter when blocking in the same fashion as [`PyOnceLock`].
Here is an example using [`PyOnceLock`] to single-initialize a runtime cache holding a `Py<PyDict>`:
```rust
# use pyo3::prelude::*;
use pyo3::sync::PyOnceLock;
use pyo3::types::PyDict;
let cache: PyOnceLock<Py<PyDict>> = PyOnceLock::new();
Python::attach(|py| {
// guaranteed to be called once and only once
cache.get_or_init(py, || PyDict::new(py).unbind())
});
```
In cases where a function must run exactly once, you can bring the [`OnceExt`] trait into scope.
The [`OnceExt`] trait adds [`OnceExt::call_once_py_attached`] and [`OnceExt::call_once_force_py_attached`] functions to the api of `std::sync::Once`, enabling use of [`Once`] in contexts where the thread is attached to the Python interpreter.
These functions are analogous to [`Once::call_once`], [`Once::call_once_force`] except they accept a [`Python<'py>`] token in addition to an `FnOnce`.
All of these functions detach from the interpreter before blocking and re-attach before executing the function, avoiding deadlocks that are possible without using the PyO3 extension traits.
Here the same example as above built using a [`Once`] instead of a
[`PyOnceLock`]:
```rust
# use pyo3::prelude::*;
use std::sync::Once;
use pyo3::sync::OnceExt;
use pyo3::types::PyDict;
struct RuntimeCache {
once: Once,
cache: Option<Py<PyDict>>
}
let mut cache = RuntimeCache {
once: Once::new(),
cache: None
};
Python::attach(|py| {
// guaranteed to be called once and only once
cache.once.call_once_py_attached(py, || {
cache.cache = Some(PyDict::new(py).unbind());
});
});
```
### `GILProtected` has been removed
[`GILProtected`] was a PyO3 type that allowed mutable access to static data by leveraging the GIL to lock concurrent access from other threads.
In free-threaded Python there is no GIL, so this type had to be replaced with alternative forms of locking.
In many cases, a type from [`std::sync::atomic`](https://doc.rust-lang.org/std/sync/atomic/) or a [`std::sync::Mutex`](https://doc.rust-lang.org/std/sync/struct.Mutex.html) was sufficient.
Before:
```rust,ignore
# fn main() {
# #[cfg(not(Py_GIL_DISABLED))] {
# use pyo3::prelude::*;
use pyo3::sync::GILProtected;
use pyo3::types::{PyDict, PyNone};
use std::cell::RefCell;
static OBJECTS: GILProtected<RefCell<Vec<Py<PyDict>>>> =
GILProtected::new(RefCell::new(Vec::new()));
Python::attach(|py| {
// stand-in for something that executes arbitrary Python code
let d = PyDict::new(py);
d.set_item(PyNone::get(py), PyNone::get(py)).unwrap();
OBJECTS.get(py).borrow_mut().push(d.unbind());
});
# }}
```
After (using a `Mutex`):
```rust
# use pyo3::prelude::*;
# fn main() {
use pyo3::types::{PyDict, PyNone};
use std::sync::Mutex;
static OBJECTS: Mutex<Vec<Py<PyDict>>> = Mutex::new(Vec::new());
Python::attach(|py| {
// stand-in for something that executes arbitrary Python code
let d = PyDict::new(py);
d.set_item(PyNone::get(py), PyNone::get(py)).unwrap();
// as with any `Mutex` usage, lock the mutex for as little time as possible
// in this case, we do it just while pushing into the `Vec`
OBJECTS.lock().unwrap().push(d.unbind());
});
# }
```
If you are executing arbitrary Python code while holding the lock, then you should import the [`MutexExt`] trait and use the `lock_py_attached` method instead of `lock`.
This ensures that global synchronization events started by the Python runtime can proceed, avoiding possible deadlocks with the interpreter.
[`GILProtected`]: https://docs.rs/pyo3/0.22/pyo3/sync/struct.GILProtected.html
[`MutexExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.MutexExt.html
[`Once`]: https://doc.rust-lang.org/stable/std/sync/struct.Once.html
[`Once::call_once`]: https://doc.rust-lang.org/stable/std/sync/struct.Once.html#method.call_once
[`Once::call_once_force`]: https://doc.rust-lang.org/stable/std/sync/struct.Once.html#method.call_once_force
[`OnceExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html
[`OnceExt::call_once_py_attached`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html#tymethod.call_once_py_attached
[`OnceExt::call_once_force_py_attached`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceExt.html#tymethod.call_once_force_py_attached
[`OnceLockExt`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceLockExt.html
[`OnceLockExt::get_or_init_py_attached`]: {{#PYO3_DOCS_URL}}/pyo3/sync/trait.OnceLockExt.html#tymethod.get_or_init_py_attached
[`OnceLock`]: https://doc.rust-lang.org/stable/std/sync/struct.OnceLock.html
[`Python::detach`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.detach
[`Python::attach`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.attach
[`Python<'py>`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html
[`PyOnceLock`]: {{#PYO3_DOCS_URL}}/pyo3/sync/struct.PyOnceLock.html
[`threading`]: https://docs.python.org/3/library/threading.html
+1
View File
@@ -0,0 +1 @@
# Calling Python functions
+272
View File
@@ -0,0 +1,272 @@
# Python functions
The `#[pyfunction]` attribute is used to define a Python function from a Rust function.
Once defined, the function needs to be added to a [module](./module.md).
The following example defines a function called `double` in a Python module called `my_extension`:
```rust,no_run
#[pyo3::pymodule]
mod my_extension {
use pyo3::prelude::*;
#[pyfunction]
fn double(x: usize) -> usize {
x * 2
}
}
```
This chapter of the guide explains full usage of the `#[pyfunction]` attribute.
In this first section, the following topics are covered:
- [Function options](#function-options)
- [`#[pyo3(name = "...")]`](#name)
- [`#[pyo3(signature = (...))]`](#signature)
- [`#[pyo3(text_signature = "...")]`](#text_signature)
- [`#[pyo3(pass_module)]`](#pass_module)
- [`#[pyo3(warn(message = "...", category = ...))]`](#warn)
- [Per-argument options](#per-argument-options)
- [Advanced function patterns](#advanced-function-patterns)
There are also additional sections on the following topics:
- [Function Signatures](./function/signature.md)
- [Error Handling](./function/error-handling.md)
## Function options
The `#[pyo3]` attribute can be used to modify properties of the generated Python function.
It can take any combination of the following options:
- <a id="name"></a> `#[pyo3(name = "...")]`
Overrides the name exposed to Python.
In the following example, the Rust function `no_args_py` will be added to the Python module `module_with_functions` as the Python function `no_args`:
```rust
# use pyo3::prelude::*;
#[pyo3::pymodule]
mod module_with_functions {
use pyo3::prelude::*;
#[pyfunction]
#[pyo3(name = "no_args")]
fn no_args_py() -> usize {
42
}
}
# Python::attach(|py| {
# let m = pyo3::wrap_pymodule!(module_with_functions)(py);
# assert!(m.getattr(py, "no_args").is_ok());
# assert!(m.getattr(py, "no_args_py").is_err());
# });
```
- <a id="signature"></a> `#[pyo3(signature = (...))]`
Defines the function signature in Python.
See [Function Signatures](./function/signature.md).
- <a id="text_signature"></a> `#[pyo3(text_signature = "...")]`
Overrides the PyO3-generated function signature visible in Python tooling (such as via [`inspect.signature`]).
See the [corresponding topic in the Function Signatures subchapter](./function/signature.md#making-the-function-signature-available-to-python).
- <a id="pass_module" ></a> `#[pyo3(pass_module)]`
Set this option to make PyO3 pass the containing module as the first argument to the function.
It is then possible to use the module in the function body.
The first argument **must** be of type `&Bound<'_, PyModule>`, `Bound<'_, PyModule>`, or `Py<PyModule>`.
The following example creates a function `pyfunction_with_module` which returns the containing module's name (i.e. `module_with_fn`):
```rust,no_run
#[pyo3::pymodule]
mod module_with_fn {
use pyo3::prelude::*;
use pyo3::types::PyString;
#[pyfunction]
#[pyo3(pass_module)]
fn pyfunction_with_module<'py>(
module: &Bound<'py, PyModule>,
) -> PyResult<Bound<'py, PyString>> {
module.name()
}
}
```
- <a id="warn"></a> `#[pyo3(warn(message = "...", category = ...))]`
This option is used to display a warning when the function is used in Python.
It is equivalent to [`warnings.warn(message, category)`](https://docs.python.org/3/library/warnings.html#warnings.warn).
The `message` parameter is a string that will be displayed when the function is called, and the `category` parameter is optional and has to be a subclass of [`Warning`](https://docs.python.org/3/library/exceptions.html#Warning).
When the `category` parameter is not provided, the warning will be defaulted to [`UserWarning`](https://docs.python.org/3/library/exceptions.html#UserWarning).
> Note: when used with `#[pymethods]`, this attribute does not work with `#[classattr]` nor `__traverse__` magic method.
The following are examples of using the `#[pyo3(warn)]` attribute:
```rust
use pyo3::prelude::*;
#[pymodule]
mod raising_warning_fn {
use pyo3::prelude::pyfunction;
use pyo3::exceptions::PyFutureWarning;
#[pyfunction]
#[pyo3(warn(message = "This is a warning message"))]
fn function_with_warning() -> usize {
42
}
#[pyfunction]
#[pyo3(warn(message = "This function is warning with FutureWarning", category = PyFutureWarning))]
fn function_with_warning_and_custom_category() -> usize {
42
}
}
# use pyo3::exceptions::{PyFutureWarning, PyUserWarning};
# use pyo3::types::{IntoPyDict, PyList};
# use pyo3::PyTypeInfo;
#
# fn catch_warning(py: Python<'_>, f: impl FnOnce(&Bound<'_, PyList>) -> ()) -> PyResult<()> {
# let warnings = py.import("warnings")?;
# let kwargs = [("record", true)].into_py_dict(py)?;
# let catch_warnings = warnings
# .getattr("catch_warnings")?
# .call((), Some(&kwargs))?;
# let list = catch_warnings.call_method0("__enter__")?.cast_into()?;
# warnings.getattr("simplefilter")?.call1(("always",))?; // show all warnings
# f(&list);
# catch_warnings
# .call_method1("__exit__", (py.None(), py.None(), py.None()))
# .unwrap();
# Ok(())
# }
#
# macro_rules! assert_warnings {
# ($py:expr, $body:expr, [$(($category:ty, $message:literal)),+] $(,)? ) => {
# catch_warning($py, |list| {
# $body;
# let expected_warnings = [$((<$category as PyTypeInfo>::type_object($py), $message)),+];
# assert_eq!(list.len(), expected_warnings.len());
# for (warning, (category, message)) in list.iter().zip(expected_warnings) {
# assert!(warning.getattr("category").unwrap().is(&category));
# assert_eq!(
# warning.getattr("message").unwrap().str().unwrap().to_string_lossy(),
# message
# );
# }
# }).unwrap();
# };
# }
#
# Python::attach(|py| {
# assert_warnings!(
# py,
# {
# let m = pyo3::wrap_pymodule!(raising_warning_fn)(py);
# let f1 = m.getattr(py, "function_with_warning").unwrap();
# let f2 = m.getattr(py, "function_with_warning_and_custom_category").unwrap();
# f1.call0(py).unwrap();
# f2.call0(py).unwrap();
# },
# [
# (PyUserWarning, "This is a warning message"),
# (
# PyFutureWarning,
# "This function is warning with FutureWarning"
# )
# ]
# );
# });
```
When the functions are called as the following, warnings will be displayed.
```python
import warnings
from raising_warning_fn import function_with_warning, function_with_warning_and_custom_category
function_with_warning()
function_with_warning_and_custom_category()
```
The warning output will be:
```plaintext
UserWarning: This is a warning message
FutureWarning: This function is warning with FutureWarning
```
## Per-argument options
The `#[pyo3]` attribute can be used on individual arguments to modify properties of them in the generated function.
It can take any combination of the following options:
- <a id="from_py_with"></a> `#[pyo3(from_py_with = ...)]`
Set this on an option to specify a custom function to convert the function argument from Python to the desired Rust type, instead of using the default `FromPyObject` extraction.
The function signature must be `fn(&Bound<'_, PyAny>) -> PyResult<T>` where `T` is the Rust type of the argument.
The following example uses `from_py_with` to convert the input Python object to its length:
```rust
use pyo3::prelude::*;
fn get_length(obj: &Bound<'_, PyAny>) -> PyResult<usize> {
obj.len()
}
#[pyfunction]
fn object_length(#[pyo3(from_py_with = get_length)] argument: usize) -> usize {
argument
}
# Python::attach(|py| {
# let f = pyo3::wrap_pyfunction!(object_length)(py).unwrap();
# assert_eq!(f.call1((vec![1, 2, 3],)).unwrap().extract::<usize>().unwrap(), 3);
# });
```
## Advanced function patterns
### Calling Python functions in Rust
You can pass Python `def`'d functions and built-in functions to Rust functions [`PyFunction`] corresponds to regular Python functions while [`PyCFunction`] describes built-ins such as `repr()`.
You can also use [`Bound<'_, PyAny>::is_callable`] to check if you have a callable object.
`is_callable` will return `true` for functions (including lambdas), methods and objects with a `__call__` method.
You can call the object with [`Bound<'_, PyAny>::call`] with the args as first parameter and the kwargs (or `None`) as second parameter.
There are also [`Bound<'_, PyAny>::call0`] with no args and [`Bound<'_, PyAny>::call1`] with only positional args.
### Calling Rust functions in Python
The ways to convert a Rust function into a Python object vary depending on the function:
- Named functions, e.g. `fn foo()`: add `#[pyfunction]` and then use [`wrap_pyfunction!`] to get the corresponding [`PyCFunction`]; see [`wrap_pyfunction!`] for an example.
- Anonymous functions (or closures), e.g. `foo: fn()` either:
- use a `#[pyclass]` struct which stores the function as a field and implement `__call__` to call the stored function.
- use `PyCFunction::new_closure` to create an object directly from the function.
### Accessing the FFI functions
In order to make Rust functions callable from Python, PyO3 generates an `extern "C"` function whose exact signature depends on the Rust signature. (PyO3 chooses the optimal Python argument passing convention.) It then embeds the call to the Rust function inside this FFI-wrapper function.
This wrapper handles extraction of the regular arguments and the keyword arguments from the input `PyObject`s.
The `wrap_pyfunction` macro can be used to directly get a `Bound<PyCFunction>` given a `#[pyfunction]` and a `Bound<PyModule>`: `wrap_pyfunction!(rust_fun, module)`.
[`Bound<'_, PyAny>::is_callable`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.is_callable
[`Bound<'_, PyAny>::call`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.call
[`Bound<'_, PyAny>::call0`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.call0
[`Bound<'_, PyAny>::call1`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/trait.PyAnyMethods.html#tymethod.call1
[`wrap_pyfunction!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.wrap_pyfunction.html
[`PyFunction`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFunction.html
[`PyCFunction`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyCFunction.html
[`inspect.signature`]: https://docs.python.org/3/library/inspect.html#inspect.signature
+251
View File
@@ -0,0 +1,251 @@
# Error handling
This chapter contains a little background of error handling in Rust and how PyO3 integrates this with Python exceptions.
This covers enough detail to create a `#[pyfunction]` which raises Python exceptions from errors originating in Rust.
There is a later section of the guide on [Python exceptions](../exception.md) which covers exception types in more detail.
## Representing Python exceptions
Rust code uses the generic [`Result<T, E>`] enum to propagate errors.
The error type `E` is chosen by the code author to describe the possible errors which can happen.
PyO3 has the [`PyErr`] type which represents a Python exception.
If a PyO3 API could result in a Python exception being raised, the return type of that `API` will be [`PyResult<T>`], which is an alias for the type `Result<T, PyErr>`.
In summary:
- When Python exceptions are raised and caught by PyO3, the exception will be stored in the `Err` variant of the `PyResult`.
- Passing Python exceptions through Rust code then uses all the "normal" techniques such as the `?` operator, with `PyErr` as the error type.
- Finally, when a `PyResult` crosses from Rust back to Python via PyO3, if the result is an `Err` variant the contained exception will be raised.
(There are many great tutorials on Rust error handling and the `?` operator, so this guide will not go into detail on Rust-specific topics.)
## Raising an exception from a function
As indicated in the previous section, when a `PyResult` containing an `Err` crosses from Rust to Python, PyO3 will raise the exception contained within.
Accordingly, to raise an exception from a `#[pyfunction]`, change the return type `T` to `PyResult<T>`.
When the function returns an `Err` it will raise a Python exception. (Other `Result<T, E>` types can be used as long as the error `E` has a `From` conversion for `PyErr`, see [custom Rust error types](#custom-rust-error-types) below.)
This also works for functions in `#[pymethods]`.
For example, the following `check_positive` function raises a `ValueError` when the input is negative:
```rust
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
#[pyfunction]
fn check_positive(x: i32) -> PyResult<()> {
if x < 0 {
Err(PyValueError::new_err("x is negative"))
} else {
Ok(())
}
}
#
# fn main(){
# Python::attach(|py|{
# let fun = pyo3::wrap_pyfunction!(check_positive, py).unwrap();
# fun.call1((-1,)).unwrap_err();
# fun.call1((1,)).unwrap();
# });
# }
```
All built-in Python exception types are defined in the [`pyo3::exceptions`] module.
They have a `new_err` constructor to directly build a `PyErr`, as seen in the example above.
## Custom Rust error types
PyO3 will automatically convert a `Result<T, E>` returned by a `#[pyfunction]` into a `PyResult<T>` as long as there is an implementation of `std::from::From<E> for PyErr`.
Many error types in the Rust standard library have a [`From`] conversion defined in this way.
If the type `E` you are handling is defined in a third-party crate, see the section on [foreign rust error types](#foreign-rust-error-types) below for ways to work with this error.
The following example makes use of the implementation of `From<ParseIntError> for PyErr` to raise exceptions encountered when parsing strings as integers:
```rust
# use pyo3::prelude::*;
use std::num::ParseIntError;
#[pyfunction]
fn parse_int(x: &str) -> Result<usize, ParseIntError> {
x.parse()
}
# fn main() {
# Python::attach(|py| {
# let fun = pyo3::wrap_pyfunction!(parse_int, py).unwrap();
# let value: usize = fun.call1(("5",)).unwrap().extract().unwrap();
# assert_eq!(value, 5);
# });
# }
```
When passed a string which doesn't contain a floating-point number, the exception raised will look like the below:
```python
>>> parse_int("bar")
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ValueError: invalid digit found in string
```
As a more complete example, the following snippet defines a Rust error named `CustomIOError`.
It then defines a `From<CustomIOError> for PyErr`, which returns a `PyErr` representing Python's `OSError`.
Therefore, it can use this error in the result of a `#[pyfunction]` directly, relying on the conversion if it has to be propagated into a Python exception.
```rust
use pyo3::exceptions::PyOSError;
use pyo3::prelude::*;
use std::fmt;
#[derive(Debug)]
struct CustomIOError;
impl std::error::Error for CustomIOError {}
impl fmt::Display for CustomIOError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "Oh no!")
}
}
impl std::convert::From<CustomIOError> for PyErr {
fn from(err: CustomIOError) -> PyErr {
PyOSError::new_err(err.to_string())
}
}
pub struct Connection {/* ... */}
fn bind(addr: String) -> Result<Connection, CustomIOError> {
if &addr == "0.0.0.0" {
Err(CustomIOError)
} else {
Ok(Connection{ /* ... */})
}
}
#[pyfunction]
fn connect(s: String) -> Result<(), CustomIOError> {
bind(s)?;
// etc.
Ok(())
}
fn main() {
Python::attach(|py| {
let fun = pyo3::wrap_pyfunction!(connect, py).unwrap();
let err = fun.call1(("0.0.0.0",)).unwrap_err();
assert!(err.is_instance_of::<PyOSError>(py));
});
}
```
If lazy construction of the Python exception instance is desired, the [`PyErrArguments`]({{#PYO3_DOCS_URL}}/pyo3/trait.PyErrArguments.html) trait can be implemented instead of `From`.
In that case, actual exception argument creation is delayed until the `PyErr` is needed.
A final note is that any errors `E` which have a `From` conversion can be used with the `?` ("try") operator with them.
An alternative implementation of the above `parse_int` which instead returns `PyResult` is below:
```rust
use pyo3::prelude::*;
fn parse_int(s: String) -> PyResult<usize> {
let x = s.parse()?;
Ok(x)
}
#
# use pyo3::exceptions::PyValueError;
#
# fn main() {
# Python::attach(|py| {
# assert_eq!(parse_int(String::from("1")).unwrap(), 1);
# assert_eq!(parse_int(String::from("1337")).unwrap(), 1337);
#
# assert!(parse_int(String::from("-1"))
# .unwrap_err()
# .is_instance_of::<PyValueError>(py));
# assert!(parse_int(String::from("foo"))
# .unwrap_err()
# .is_instance_of::<PyValueError>(py));
# assert!(parse_int(String::from("13.37"))
# .unwrap_err()
# .is_instance_of::<PyValueError>(py));
# })
# }
```
## Foreign Rust error types
The Rust compiler will not permit implementation of traits for types outside of the crate where the type is defined. (This is known as the "orphan rule".)
Given a type `OtherError` which is defined in third-party code, there are two main strategies available to integrate it with PyO3:
- Create a newtype wrapper, e.g. `MyOtherError`.
Then implement `From<MyOtherError> for PyErr` (or `PyErrArguments`), as well as `From<OtherError>` for `MyOtherError`.
- Use Rust's Result combinators such as `map_err` to write code freely to convert `OtherError` into whatever is needed.
This requires boilerplate at every usage however gives unlimited flexibility.
To detail the newtype strategy a little further, the key trick is to return `Result<T, MyOtherError>` from the `#[pyfunction]`.
This means that PyO3 will make use of `From<MyOtherError> for PyErr` to create Python exceptions while the `#[pyfunction]` implementation can use `?` to convert `OtherError` to `MyOtherError` automatically.
The following example demonstrates this for some imaginary third-party crate `some_crate` with a function `get_x` returning `Result<i32, OtherError>`:
```rust
# mod some_crate {
# pub struct OtherError(());
# impl OtherError {
# pub fn message(&self) -> &'static str { "some error occurred" }
# }
# pub fn get_x() -> Result<i32, OtherError> { Ok(5) }
# }
use pyo3::prelude::*;
use pyo3::exceptions::PyValueError;
use some_crate::{OtherError, get_x};
struct MyOtherError(OtherError);
impl From<MyOtherError> for PyErr {
fn from(error: MyOtherError) -> Self {
PyValueError::new_err(error.0.message())
}
}
impl From<OtherError> for MyOtherError {
fn from(other: OtherError) -> Self {
Self(other)
}
}
#[pyfunction]
fn wrapped_get_x() -> Result<i32, MyOtherError> {
// get_x is a function returning Result<i32, OtherError>
let x: i32 = get_x()?;
Ok(x)
}
# fn main() {
# Python::attach(|py| {
# let fun = pyo3::wrap_pyfunction!(wrapped_get_x, py).unwrap();
# let value: usize = fun.call0().unwrap().extract().unwrap();
# assert_eq!(value, 5);
# });
# }
```
## Notes
In Python 3.11 and up, notes can be added to Python exceptions to provide additional debugging information when printing the exception.
In PyO3, you can use the `add_note` method on `PyErr` to accomplish this functionality.
[`From`]: https://doc.rust-lang.org/stable/std/convert/trait.From.html
[`Result<T, E>`]: https://doc.rust-lang.org/stable/std/result/enum.Result.html
[`PyResult<T>`]: {{#PYO3_DOCS_URL}}/pyo3/prelude/type.PyResult.html
[`PyErr`]: {{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html
[`pyo3::exceptions`]: {{#PYO3_DOCS_URL}}/pyo3/exceptions/index.html
+300
View File
@@ -0,0 +1,300 @@
# Function signatures
The `#[pyfunction]` attribute also accepts parameters to control how the generated Python function accepts arguments.
Just like in Python, arguments can be positional-only, keyword-only, or accept either.
`*args` lists and `**kwargs` dicts can also be accepted.
These parameters also work for `#[pymethods]` which will be introduced in the [Python Classes](../class.md) section of the guide.
Like Python, by default PyO3 accepts all arguments as either positional or keyword arguments.
All arguments are required by default.
This behaviour can be configured by the `#[pyo3(signature = (...))]` option which allows writing a signature in Python syntax.
This section of the guide goes into detail about use of the `#[pyo3(signature = (...))]` option and its related option `#[pyo3(text_signature = "...")]`
## Using `#[pyo3(signature = (...))]`
For example, below is a function that accepts arbitrary keyword arguments (`**kwargs` in Python syntax) and returns the number that was passed:
```rust,no_run
#[pyo3::pymodule]
mod module_with_functions {
use pyo3::prelude::*;
use pyo3::types::PyDict;
#[pyfunction]
#[pyo3(signature = (**kwds))]
fn num_kwds(kwds: Option<&Bound<'_, PyDict>>) -> usize {
kwds.map_or(0, |dict| dict.len())
}
}
```
Just like in Python, the following constructs can be part of the signature::
- `/`: positional-only arguments separator, each parameter defined before `/` is a positional-only parameter.
- `*`: var arguments separator, each parameter defined after `*` is a keyword-only parameter.
- `*args`: "args" is var args.
Type of the `args` parameter has to be `&Bound<'_, PyTuple>`.
- `**kwargs`: "kwargs" receives keyword arguments.
The type of the `kwargs` parameter has to be `Option<&Bound<'_, PyDict>>`.
- `arg=Value`: arguments with default value.
If the `arg` argument is defined after var arguments, it is treated as a keyword-only argument.
Note that `Value` has to be valid rust code, PyO3 just inserts it into the generated code unmodified.
Example:
```rust,no_run
# use pyo3::prelude::*;
use pyo3::types::{PyDict, PyTuple};
#
# #[pyclass]
# struct MyClass {
# num: i32,
# }
#[pymethods]
impl MyClass {
#[new]
#[pyo3(signature = (num=-1))]
fn new(num: i32) -> Self {
MyClass { num }
}
#[pyo3(signature = (num=10, *py_args, name="Hello", **py_kwargs))]
fn method(
&mut self,
num: i32,
py_args: &Bound<'_, PyTuple>,
name: &str,
py_kwargs: Option<&Bound<'_, PyDict>>,
) -> String {
let num_before = self.num;
self.num = num;
format!(
"num={} (was previously={}), py_args={:?}, name={}, py_kwargs={:?} ",
num, num_before, py_args, name, py_kwargs,
)
}
fn make_change(&mut self, num: i32) -> PyResult<String> {
self.num = num;
Ok(format!("num={}", self.num))
}
}
```
Arguments of type `Python` must not be part of the signature:
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
#[pyfunction]
#[pyo3(signature = (lambda))]
pub fn simple_python_bound_function(py: Python<'_>, lambda: Py<PyAny>) -> PyResult<()> {
Ok(())
}
```
N.B. the position of the `/` and `*` arguments (if included) control the system of handling positional and keyword arguments.
In Python:
```python
import mymodule
mc = mymodule.MyClass()
print(mc.method(44, False, "World", 666, x=44, y=55))
print(mc.method(num=-1, name="World"))
print(mc.make_change(44))
```
Produces output:
```text
num=44 (was previously=-1), py_args=(False, 'World', 666), name=Hello, py_kwargs=Some({'x': 44, 'y': 55})
num=-1 (was previously=44), py_args=(), name=World, py_kwargs=None
num=44
```
> [!NOTE]
> To use keywords like `struct` as a function argument, use "raw identifier" syntax `r#struct` in both the signature and the function definition:
>
> ```rust,no_run
> # #![allow(dead_code)]
> # use pyo3::prelude::*;
> #[pyfunction(signature = (r#struct = "foo"))]
> fn function_with_keyword(r#struct: &str) {
> # let _ = r#struct;
> /* ... */
> }
> ```
## Making the function signature available to Python
The function signature is exposed to Python via the `__text_signature__` attribute.
PyO3 automatically generates this for every `#[pyfunction]` and all `#[pymethods]` directly from the Rust function, taking into account any override done with the `#[pyo3(signature = (...))]` option.
This automatic generation can only display the value of default arguments for strings, integers, boolean types, and `None`.
Any other default arguments will be displayed as `...`. (`.pyi` type stub files commonly also use `...` for default arguments in the same way.)
In cases where the automatically-generated signature needs adjusting, it can [be overridden](#overriding-the-generated-signature) using the `#[pyo3(text_signature)]` option.)
The example below creates a function `add` which accepts two positional-only arguments `a` and `b`, where `b` has a default value of zero.
```rust
use pyo3::prelude::*;
/// This function adds two unsigned 64-bit integers.
#[pyfunction]
#[pyo3(signature = (a, b=0, /))]
fn add(a: u64, b: u64) -> u64 {
a + b
}
#
# fn main() -> PyResult<()> {
# Python::attach(|py| {
# let fun = pyo3::wrap_pyfunction!(add, py)?;
#
# let doc: String = fun.getattr("__doc__")?.extract()?;
# assert_eq!(doc, "This function adds two unsigned 64-bit integers.");
#
# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?;
# let sig: String = inspect
# .call1((fun,))?
# .call_method0("__str__")?
# .extract()?;
#
# assert_eq!(sig, "(a, b=0, /)");
#
# Ok(())
# })
# }
```
The following IPython output demonstrates how this generated signature will be seen from Python tooling:
```text
>>> pyo3_test.add.__text_signature__
'(a, b=..., /)'
>>> pyo3_test.add?
Signature: pyo3_test.add(a, b=0, /)
Docstring: This function adds two unsigned 64-bit integers.
Type: builtin_function_or_method
```
### Overriding the generated signature
The `#[pyo3(text_signature = "(<some signature>)")]` attribute can be used to override the default generated signature.
In the snippet below, the text signature attribute is used to include the default value of `0` for the argument `b`, instead of the automatically-generated default value of `...`:
```rust
use pyo3::prelude::*;
/// This function adds two unsigned 64-bit integers.
#[pyfunction]
#[pyo3(signature = (a, b=0, /), text_signature = "(a, b=0, /)")]
fn add(a: u64, b: u64) -> u64 {
a + b
}
#
# fn main() -> PyResult<()> {
# Python::attach(|py| {
# let fun = pyo3::wrap_pyfunction!(add, py)?;
#
# let doc: String = fun.getattr("__doc__")?.extract()?;
# assert_eq!(doc, "This function adds two unsigned 64-bit integers.");
#
# let inspect = PyModule::import(py, "inspect")?.getattr("signature")?;
# let sig: String = inspect
# .call1((fun,))?
# .call_method0("__str__")?
# .extract()?;
# assert_eq!(sig, "(a, b=0, /)");
#
# Ok(())
# })
# }
```
PyO3 will include the contents of the annotation unmodified as the `__text_signature__`.
Below shows how IPython will now present this (see the default value of 0 for b):
```text
>>> pyo3_test.add.__text_signature__
'(a, b=0, /)'
>>> pyo3_test.add?
Signature: pyo3_test.add(a, b=0, /)
Docstring: This function adds two unsigned 64-bit integers.
Type: builtin_function_or_method
```
If no signature is wanted at all, `#[pyo3(text_signature = None)]` will disable the built-in signature.
The snippet below demonstrates use of this:
```rust
use pyo3::prelude::*;
/// This function adds two unsigned 64-bit integers.
#[pyfunction]
#[pyo3(signature = (a, b=0, /), text_signature = None)]
fn add(a: u64, b: u64) -> u64 {
a + b
}
#
# fn main() -> PyResult<()> {
# Python::attach(|py| {
# let fun = pyo3::wrap_pyfunction!(add, py)?;
#
# let doc: String = fun.getattr("__doc__")?.extract()?;
# assert_eq!(doc, "This function adds two unsigned 64-bit integers.");
# assert!(fun.getattr("__text_signature__")?.is_none());
#
# Ok(())
# })
# }
```
Now the function's `__text_signature__` will be set to `None`, and IPython will not display any signature in the help:
```text
>>> pyo3_test.add.__text_signature__ == None
True
>>> pyo3_test.add?
Docstring: This function adds two unsigned 64-bit integers.
Type: builtin_function_or_method
```
### Type annotations in the signature
The `signature` attribute can also contain type hints:
```rust
use pyo3::prelude::*;
#[pymodule]
pub mod example {
use pyo3::prelude::*;
#[pyfunction]
#[pyo3(signature = (arg: "list[int]") -> "list[int]")]
fn list_of_int_identity(arg: Bound<'_, PyAny>) -> Bound<'_, PyAny> {
arg
}
}
```
It enables the [work-in-progress capacity of PyO3 to autogenerate type stubs](../type-stub.md) to generate a file with the correct type hints:
```python
def list_of_int_identity(arg: "list[int]") -> "list[int]": ...
```
instead of the generic:
```python
from typing import Any
def list_of_int_identity(arg: Any) -> Any: ...
```
Note that currently type annotations must be written as strings.
Future enhancements of PyO3's typing support may include the ability to parse a limited subset of Python's type expression syntax.
+203
View File
@@ -0,0 +1,203 @@
# Installation
To get started using PyO3 you will need three things: a Rust toolchain, a Python environment, and a way to build.
We'll cover each of these below.
> [!TIP]
> If you'd like to chat to the PyO3 maintainers and other PyO3 users, consider joining the [PyO3 Discord server](https://discord.gg/33kcChzH7f). We're keen to hear about your experience getting started, so we can make PyO3 as accessible as possible for everyone!
## Rust
First, make sure you have Rust installed on your system.
If you haven't already done so, try following the instructions [on the Rust website](https://www.rust-lang.org/tools/install).
PyO3 runs on both the `stable` and `nightly` versions so you can choose whichever one fits you best.
The minimum required Rust version is 1.83.
If you can run `rustc --version` and the version is new enough you're good to go!
## Python
To use PyO3, you need at least Python 3.8.
While you can simply use the default Python interpreter on your system, it is recommended to use a virtual environment.
## Virtualenvs
While you can use any virtualenv manager you like, we recommend the use of `pyenv` in particular if you want to develop or test for multiple different Python versions, so that is what the examples in this book will use.
The installation instructions for `pyenv` can be found [in the `pyenv` GitHub repository](https://github.com/pyenv/pyenv#a-getting-pyenv). (Note: To get the `pyenv activate` and `pyenv virtualenv` commands, you will also need to install the [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv) plugin.
The [pyenv installer](https://github.com/pyenv/pyenv-installer#installation--update--uninstallation) will install both together.)
It can be useful to keep the sources used when installing using `pyenv` so that future debugging can see the original source files.
This can be done by passing the `--keep` flag as part of the `pyenv install` command.
For example:
```bash
pyenv install 3.12 --keep
```
### Building
There are a number of build and Python package management systems such as [`setuptools-rust`](https://github.com/PyO3/setuptools-rust) or [manually](./building-and-distribution.md#manual-builds).
We recommend the use of `maturin`, which you can install [as per the `maturin` documentation](https://maturin.rs/installation.html).
It is developed to work with PyO3 and provides the most "batteries included" experience, especially if you are aiming to publish to PyPI.
`maturin` is just a Python package, so you can add it in the same way you already install Python packages.
System Python:
```bash
pip install maturin --user
```
pipx:
```bash
pipx install maturin
```
pyenv:
```bash
pyenv activate pyo3
pip install maturin
```
poetry:
```bash
poetry add -G dev maturin
```
After installation, you can run `maturin --version` to check that you have correctly installed it.
## Starting a new project
First you should create the folder and virtual environment that are going to contain your new project.
Here we will use the recommended `pyenv`:
```bash
mkdir pyo3-example
cd pyo3-example
pyenv virtualenv pyo3
pyenv local pyo3
```
After this, you should install your build manager.
In this example, we will use `maturin`.
After you've activated your virtualenv, add `maturin` to it:
```bash
pip install maturin
```
Now you can initialize the new project:
```bash
maturin init
```
If `maturin` is already installed, you can create a new project using that directly as well:
```bash
maturin new -b pyo3 pyo3-example
cd pyo3-example
pyenv virtualenv pyo3
pyenv local pyo3
```
## Adding to an existing project
Sadly, `maturin` cannot currently be run in existing projects, so if you want to use Python in an existing project you basically have two options:
1. Create a new project as above and move your existing code into that project
2. Manually edit your project configuration as necessary
If you opt for the second option, here are the things you need to pay attention to:
## Cargo.toml
Make sure that the Rust crate you want to be able to access from Python is compiled into a library.
You can have a binary output as well, but the code you want to access from Python has to be in the library part.
Also, make sure that the crate type is `cdylib` and add PyO3 as a dependency as so:
```toml
# If you already have [package] information in `Cargo.toml`, you can ignore
# this section!
[package]
# `name` here is name of the package.
name = "pyo3_start"
# these are good defaults:
version = "0.1.0"
edition = "2021"
[lib]
# The name of the native library. This is the name which will be used in Python to import the
# library (i.e. `import string_sum`). If you change this, you must also change the name of the
# `#[pymodule]` in `src/lib.rs`.
name = "pyo3_example"
# "cdylib" is necessary to produce a shared library for Python to import from.
crate-type = ["cdylib"]
[dependencies]
pyo3 = { {{#PYO3_CRATE_VERSION}} }
```
## pyproject.toml
You should also create a `pyproject.toml` with the following contents:
```toml
[build-system]
requires = ["maturin>=1.9.4,<2"]
build-backend = "maturin"
[project]
name = "pyo3_example"
requires-python = ">=3.8"
classifiers = [
"Programming Language :: Rust",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
]
```
## Running code
After this you can setup Rust code to be available in Python as below; for example, you can place this code in `src/lib.rs`:
```rust,no_run
/// A Python module implemented in Rust. The name of this function must match
/// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to
/// import the module.
#[pyo3::pymodule]
mod pyo3_example {
use pyo3::prelude::*;
/// Formats the sum of two numbers as string.
#[pyfunction]
fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
Ok((a + b).to_string())
}
}
```
Now you can run `maturin develop` to prepare the Python package, after which you can use it like so:
```bash
$ maturin develop
# lots of progress output as maturin runs the compilation...
$ python
>>> import pyo3_example
>>> pyo3_example.sum_as_string(5, 20)
'25'
```
By default, `maturin develop` builds your Rust code with Cargo's default [`dev` profile](https://doc.rust-lang.org/cargo/reference/profiles.html#default-profiles), which includes debug information and limited optimizations.
When checking runtime performance, run `maturin develop --release` to build with Cargo's `release` profile and enable optimizations.
For more instructions on how to use Python code from Rust, see the [Python from Rust](python-from-rust.md) page.
## Maturin Import Hook
In development, any changes in the code would require running `maturin develop` before testing.
To streamline the development process, you may want to install [Maturin Import Hook](https://github.com/PyO3/maturin-import-hook) which will run `maturin develop` automatically when the library with code changes is being imported.
+45
View File
@@ -0,0 +1,45 @@
# Glossary
Many of the terms used in this guide are common to the Python or Rust ecosystem, and accordingly can be found in the [Python Glossary](https://docs.python.org/3/glossary.html), [Rust Glossary](https://doc.rust-lang.org/reference/glossary.html), or [Cargo Glossary](https://doc.rust-lang.org/cargo/appendix/glossary.html).
Below are some terms that are particularly relevant to PyO3 and either are not found in the above glossaries or have a specific meaning in the context of PyO3:
attached
: To call into the Python interpreter from Rust, the Rust thread must have an associated Python thread state which is "attached" to the Python interpreter.
This is a safety invariant required by the Python C API, which ensures that the Python interpreter can handle exceptions, avoid data races with the garbage collector, etc.
The [`Python::attach`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.attach) method is used to attach the current thread to the Python interpreter and obtain a `Python` token which can be used to call Python APIs.
extension module
: A Python module which is implemented using native code (e.g. C, C++ or Rust) instead of Python.
See also [CPython's documentation on creating extension modules](https://docs.python.org/3/extending/extending.html).
GIL-enabled Python
: The (current, as of Python 3.14) default build of Python, which depends on the GIL for thread safety.
This was historically the only thread safety strategy before the introduction of free-threaded Python.
Python token
: PyO3's [`Python<'py>`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html) type, which can only exist when the current thread is [attached](#attached) to the Python interpreter.
This type is used to call Python APIs and ensures that the safety invariants required by the Python C API are upheld.
smart pointer
: Pointers which automatically manage the memory they point to.
In particular this guide refers to [PyO3's smart pointers](types.md#pyo3s-smart-pointers) `Py<T>`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>` which point to Python objects and use Python reference counting to ensure correct memory management.
See also the [Rust book's chapter on smart pointers](https://doc.rust-lang.org/book/ch15-00-smart-pointers.html).
wheel
: A precompiled Python package format.
Wheels are the standard way to distribute native code in Python packages for common operating systems and Python versions, which avoid the need for users to compile from source.
The alternative is known as the "source distribution", or "sdist", which requires users to compile the code as part of package installation.
<!-- external-glossary-links
data races: https://docs.python.org/3/glossary.html#term-data-race
free-threading: https://docs.python.org/3/glossary.html#term-free-threading
free-threaded Python: https://docs.python.org/3/glossary.html#term-free-threaded-build
garbage collection: https://docs.python.org/3/glossary.html#term-garbage-collection
GIL: https://docs.python.org/3/glossary.html#term-GIL
global interpreter lock: https://docs.python.org/3/glossary.html#term-global-interpreter-lock
thread state: https://docs.python.org/3/glossary.html#term-thread-state
virtual environment: https://docs.python.org/3/glossary.html#term-virtual-environment
-->
+17
View File
@@ -0,0 +1,17 @@
# The PyO3 user guide
Welcome to the PyO3 user guide!
This book is a companion to [PyO3's API docs](https://docs.rs/pyo3).
It contains examples and documentation to explain all of PyO3's use cases in detail.
The rough order of material in this user guide is as follows:
1. Getting started
2. Wrapping Rust code for use from Python
3. How to use Python code from Rust
4. Remaining topics which go into advanced concepts in detail
Please choose from the chapters on the left to jump to individual topics, or continue below to start with PyO3's README.
<hr style="opacity:0.2">
{{#include ../../README.md}}
+2955
View File
File diff suppressed because it is too large. Load diff
+176
View File
@@ -0,0 +1,176 @@
# Python modules
You can create a module using `#[pymodule]`:
```rust,no_run
# mod declarative_module_basic_test {
use pyo3::prelude::*;
#[pyfunction]
fn double(x: usize) -> usize {
x * 2
}
/// This module is implemented in Rust.
#[pymodule]
mod my_extension {
use pyo3::prelude::*;
#[pymodule_export]
use super::double; // The double function is made available from Python, works also with classes
#[pyfunction] // Inline definition of a pyfunction, also made available to Python
fn triple(x: usize) -> usize {
x * 3
}
}
# }
```
The `#[pymodule]` procedural macro takes care of creating the initialization function of your module and exposing it to Python.
The module's name defaults to the name of the Rust module.
You can override the module name by using `#[pyo3(name = "custom_name")]`:
```rust,no_run
# mod declarative_module_custom_name_test {
use pyo3::prelude::*;
#[pyfunction]
fn double(x: usize) -> usize {
x * 2
}
#[pymodule(name = "custom_name")]
mod my_extension {
#[pymodule_export]
use super::double;
}
# }
```
The name of the module must match the name of the `.so` or `.pyd` file.
Otherwise, you will get an import error in Python with the following message: `ImportError: dynamic module does not define module export function (PyInit_name_of_your_module)`
To import the module, either:
- copy the shared library as described in [Manual builds](building-and-distribution.md#manual-builds), or
- use a tool, e.g. `maturin develop` with [maturin](https://github.com/PyO3/maturin) or
`python setup.py develop` with [setuptools-rust](https://github.com/PyO3/setuptools-rust).
## Documentation
The [Rust doc comments](https://doc.rust-lang.org/stable/book/ch03-04-comments.html) of the Rust module will be applied automatically as the Python docstring of your module.
For example, building off of the above code, this will print `This module is implemented in Rust.`:
```python
import my_extension
print(my_extension.__doc__)
```
## Python submodules
You can create a module hierarchy within a single extension module by just `use`ing modules like functions or classes.
For example, you could define the modules `parent_module` and `parent_module.child_module`:
```rust
use pyo3::prelude::*;
#[pymodule]
mod parent_module {
#[pymodule_export]
use super::child_module;
}
#[pymodule]
mod child_module {
#[pymodule_export]
use super::func;
}
#[pyfunction]
fn func() -> String {
"func".to_string()
}
#
# fn main() {
# Python::attach(|py| {
# use pyo3::wrap_pymodule;
# use pyo3::types::IntoPyDict;
# let parent_module = wrap_pymodule!(parent_module)(py);
# let ctx = [("parent_module", parent_module)].into_py_dict(py).unwrap();
#
# py.run(c"assert parent_module.child_module.func() == 'func'", None, Some(&ctx)).unwrap();
# })
# }
```
Note that this does not define a package, so this won’t allow Python code to directly import submodules by using `from parent_module import child_module`.
For more information, see [#759](https://github.com/PyO3/pyo3/issues/759) and [#1517](https://github.com/PyO3/pyo3/issues/1517#issuecomment-808664021).
You can provide the `submodule` argument to `#[pymodule()]` for modules that are not top-level modules in order for them to properly generate the `#[pyclass]` `module` attribute automatically.
## Inline declaration
It is possible to declare functions, classes, sub-modules and constants inline in a module:
For example:
```rust,no_run
# mod declarative_module_test {
#[pyo3::pymodule]
mod my_extension {
use pyo3::prelude::*;
#[pymodule_export]
const PI: f64 = std::f64::consts::PI; // Exports PI constant as part of the module
#[pyfunction] // This will be part of the module
fn double(x: usize) -> usize {
x * 2
}
#[pyclass] // This will be part of the module
struct Unit;
#[pymodule]
mod submodule {
// This is a submodule
use pyo3::prelude::*;
#[pyclass]
struct Nested;
}
}
# }
```
In this case, `#[pymodule]` macro automatically sets the `module` attribute of the `#[pyclass]` macros declared inside of it with its name.
For nested modules, the name of the parent module is automatically added.
In the previous example, the `Nested` class will have for `module` `my_extension.submodule`.
## Procedural initialization
If the macros provided by PyO3 are not enough, it is possible to run code at the module initialization:
```rust,no_run
# mod procedural_module_test {
#[pyo3::pymodule]
mod my_extension {
use pyo3::prelude::*;
#[pyfunction]
fn double(x: usize) -> usize {
x * 2
}
#[pymodule_init]
fn init(m: &Bound<'_, PyModule>) -> PyResult<()> {
// Arbitrary code to run at the module initialization
m.add("double2", m.getattr("double")?)
}
}
# }
```
+183
View File
@@ -0,0 +1,183 @@
# Parallelism
Historically, CPython was limited by the global interpreter lock (GIL), which only allowed a single thread to drive the Python interpreter at a time.
This made threading in Python a bad fit for [CPU-bound](https://en.wikipedia.org/wiki/CPU-bound) tasks and often forced developers to accept the overhead of multiprocessing.
Rust is well-suited to multithreaded code, and libraries like [`rayon`] can help you leverage safe parallelism with minimal effort.
The [`Python::detach`] method can be used to allow the Python interpreter to do other work while the Rust work is ongoing.
To enable full parallelism in your application, consider also using [free-threaded Python](./free-threading.md) which is supported since Python 3.14.
## Parallelism under the Python GIL
Let's take a look at our [word-count](https://github.com/PyO3/pyo3/blob/main/examples/word-count/src/lib.rs) example, where we have a `search` function that utilizes the [`rayon`] crate to count words in parallel.
```rust,no_run
# #![allow(dead_code)]
use pyo3::prelude::*;
// These traits let us use `par_lines` and `map`.
use rayon::str::ParallelString;
use rayon::iter::ParallelIterator;
/// Count the occurrences of needle in line, case insensitive
fn count_line(line: &str, needle: &str) -> usize {
let mut total = 0;
for word in line.split(' ') {
if word == needle {
total += 1;
}
}
total
}
#[pyfunction]
fn search(contents: &str, needle: &str) -> usize {
contents
.par_lines()
.map(|line| count_line(line, needle))
.sum()
}
```
But let's assume you have a long running Rust function which you would like to execute several times in parallel.
For the sake of example let's take a sequential version of the word count:
```rust,no_run
# #![allow(dead_code)]
# fn count_line(line: &str, needle: &str) -> usize {
# let mut total = 0;
# for word in line.split(' ') {
# if word == needle {
# total += 1;
# }
# }
# total
# }
#
fn search_sequential(contents: &str, needle: &str) -> usize {
contents.lines().map(|line| count_line(line, needle)).sum()
}
```
To enable parallel execution of this function, the [`Python::detach`] method can be used to temporarily release the GIL, thus allowing other Python threads to run.
We then have a function exposed to the Python runtime which calls `search_sequential` inside a closure passed to [`Python::detach`] to enable true parallelism:
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
#
# fn count_line(line: &str, needle: &str) -> usize {
# let mut total = 0;
# for word in line.split(' ') {
# if word == needle {
# total += 1;
# }
# }
# total
# }
#
# fn search_sequential(contents: &str, needle: &str) -> usize {
# contents.lines().map(|line| count_line(line, needle)).sum()
# }
#[pyfunction]
fn search_sequential_detached(py: Python<'_>, contents: &str, needle: &str) -> usize {
py.detach(|| search_sequential(contents, needle))
}
```
Now Python threads can use more than one CPU core, resolving the limitation which usually makes multi-threading in Python only good for IO-bound tasks:
```Python
from concurrent.futures import ThreadPoolExecutor
from word_count import search_sequential_detached
executor = ThreadPoolExecutor(max_workers=2)
future_1 = executor.submit(
word_count.search_sequential_detached, contents, needle
)
future_2 = executor.submit(
word_count.search_sequential_detached, contents, needle
)
result_1 = future_1.result()
result_2 = future_2.result()
```
## Benchmark
Let's benchmark the `word-count` example to verify that we really did unlock parallelism with PyO3.
We are using `pytest-benchmark` to benchmark four word count functions:
1. Pure Python version
2. Rust parallel version
3. Rust sequential version
4. Rust sequential version executed twice with two Python threads
The benchmark script can be found [in the PyO3 GitHub repository](https://github.com/PyO3/pyo3/blob/main/examples/word-count/tests/test_word_count.py), and we can run `nox` in the `word-count` folder to benchmark these functions.
While the results of the benchmark of course depend on your machine, the relative results should be similar to this (mid 2020):
```text
-------------------------------------------------------------------------------------------------- benchmark: 4 tests -------------------------------------------------------------------------------------------------
Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
test_word_count_rust_parallel 1.7315 (1.0) 4.6495 (1.0) 1.9972 (1.0) 0.4299 (1.0) 1.8142 (1.0) 0.2049 (1.0) 40;46 500.6943 (1.0) 375 1
test_word_count_rust_sequential 7.3348 (4.24) 10.3556 (2.23) 8.0035 (4.01) 0.7785 (1.81) 7.5597 (4.17) 0.8641 (4.22) 26;5 124.9457 (0.25) 121 1
test_word_count_rust_sequential_twice_with_threads 7.9839 (4.61) 10.3065 (2.22) 8.4511 (4.23) 0.4709 (1.10) 8.2457 (4.55) 0.3927 (1.92) 17;17 118.3274 (0.24) 114 1
test_word_count_python_sequential 27.3985 (15.82) 45.4527 (9.78) 28.9604 (14.50) 4.1449 (9.64) 27.5781 (15.20) 0.4638 (2.26) 3;5 34.5299 (0.07) 35 1
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
```
You can see that the Python threaded version is not much slower than the Rust sequential version, which means compared to an execution on a single CPU core the speed has doubled.
## Sharing Python objects between Rust threads
In the example above we made a Python interface to a low-level rust function, and then leveraged the python `threading` module to run the low-level function in parallel.
It is also possible to spawn threads in Rust that acquire the GIL and operate on Python objects.
However, care must be taken to avoid writing code that deadlocks with the GIL in these cases.
- Note: This example is meant to illustrate how to drop and re-acquire the GIL
to avoid creating deadlocks. Unless the spawned threads subsequently
release the GIL or you are using the free-threaded build of CPython, you
will not see any speedups due to multi-threaded parallelism using `rayon`
to parallelize code that acquires and holds the GIL for the entire
execution of the spawned thread.
In the example below, we share a `Vec` of User ID objects defined using the `pyclass` macro and spawn threads to process the collection of data into a `Vec` of booleans based on a predicate using a `rayon` parallel iterator:
```rust,no_run
use pyo3::prelude::*;
// These traits let us use int_par_iter and map
use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
#[pyclass]
struct UserID {
id: i64,
}
let allowed_ids: Vec<bool> = Python::attach(|outer_py| {
let instances: Vec<Py<UserID>> = (0..10).map(|x| Py::new(outer_py, UserID { id: x }).unwrap()).collect();
outer_py.detach(|| {
instances.par_iter().map(|instance| {
Python::attach(|inner_py| {
instance.borrow(inner_py).id > 5
})
}).collect()
})
});
assert!(allowed_ids.into_iter().filter(|b| *b).count() == 4);
```
It's important to note that there is an `outer_py` Python token as well as an `inner_py` token.
Sharing Python tokens between threads is not allowed and threads must individually attach to the interpreter to access data wrapped by a Python object.
It's also important to see that this example uses [`Python::detach`] to wrap the code that spawns OS threads via `rayon`.
If this example didn't use `detach`, a `rayon` worker thread would block on acquiring the GIL while a thread that owns the GIL spins forever waiting for the result of the `rayon` thread.
Calling `detach` allows the GIL to be released in the thread collecting the results from the worker threads.
You should always call `detach` in situations that spawn worker threads, but especially so in cases where worker threads need to acquire the GIL, to prevent deadlocks.
[`Python::detach`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.detach
[`rayon`]: https://github.com/rayon-rs/rayon
+181
View File
@@ -0,0 +1,181 @@
# Performance
To achieve the best possible performance, it is useful to be aware of several tricks and sharp edges concerning PyO3's API.
## `extract` versus `cast`
Pythonic API implemented using PyO3 are often polymorphic, i.e. they will accept `&Bound<'_, PyAny>` and try to turn this into multiple more concrete types to which the requested operation is applied.
This often leads to chains of calls to `extract`, e.g.
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
# use pyo3::{exceptions::PyTypeError, types::PyList};
fn frobnicate_list<'py>(list: &Bound<'_, PyList>) -> PyResult<Bound<'py, PyAny>> {
todo!()
}
fn frobnicate_vec<'py>(vec: Vec<Bound<'py, PyAny>>) -> PyResult<Bound<'py, PyAny>> {
todo!()
}
#[pyfunction]
fn frobnicate<'py>(value: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
if let Ok(list) = value.extract::<Bound<'_, PyList>>() {
frobnicate_list(&list)
} else if let Ok(vec) = value.extract::<Vec<Bound<'_, PyAny>>>() {
frobnicate_vec(vec)
} else {
Err(PyTypeError::new_err("Cannot frobnicate that type."))
}
}
```
This suboptimal as the `FromPyObject<T>` trait requires `extract` to have a `Result<T, PyErr>` return type.
For native types like `PyList`, it faster to use `cast` (which `extract` calls internally) when the error value is ignored.
This avoids the costly conversion of a `PyDowncastError` to a `PyErr` required to fulfil the `FromPyObject` contract, i.e.
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
# use pyo3::{exceptions::PyTypeError, types::PyList};
# fn frobnicate_list<'py>(list: &Bound<'_, PyList>) -> PyResult<Bound<'py, PyAny>> { todo!() }
# fn frobnicate_vec<'py>(vec: Vec<Bound<'py, PyAny>>) -> PyResult<Bound<'py, PyAny>> { todo!() }
#
#[pyfunction]
fn frobnicate<'py>(value: &Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
// Use `cast` instead of `extract` as turning `PyDowncastError` into `PyErr` is quite costly.
if let Ok(list) = value.cast::<PyList>() {
frobnicate_list(list)
} else if let Ok(vec) = value.extract::<Vec<Bound<'_, PyAny>>>() {
frobnicate_vec(vec)
} else {
Err(PyTypeError::new_err("Cannot frobnicate that type."))
}
}
```
## Access to Bound implies access to Python token
Calling `Python::attach` is effectively a no-op when we're already attached to the interpreter, but checking that this is the case still has a cost.
If an existing Python token can not be accessed, for example when implementing a pre-existing trait, but a Python-bound reference is available, this cost can be avoided by exploiting that access to Python-bound reference gives zero-cost access to a Python token via `Bound::py`.
For example, instead of writing
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
# use pyo3::types::PyList;
struct Foo(Py<PyList>);
struct FooBound<'py>(Bound<'py, PyList>);
impl PartialEq<Foo> for FooBound<'_> {
fn eq(&self, other: &Foo) -> bool {
Python::attach(|py| {
let len = other.0.bind(py).len();
self.0.len() == len
})
}
}
```
use the more efficient
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
# use pyo3::types::PyList;
# struct Foo(Py<PyList>);
# struct FooBound<'py>(Bound<'py, PyList>);
#
impl PartialEq<Foo> for FooBound<'_> {
fn eq(&self, other: &Foo) -> bool {
// Access to `&Bound<'py, PyAny>` implies access to `Python<'py>`.
let py = self.0.py();
let len = other.0.bind(py).len();
self.0.len() == len
}
}
```
## Calling Python callables (`__call__`)
CPython support multiple calling protocols: [`tp_call`] and [`vectorcall`].
[`vectorcall`] is a more efficient protocol unlocking faster calls.
PyO3 will try to dispatch Python `call`s using the [`vectorcall`] calling convention to archive maximum performance if possible and falling back to [`tp_call`] otherwise.
This is implemented using the (internal) `PyCallArgs` trait.
It defines how Rust types can be used as Python `call` arguments.
This trait is currently implemented for
- Rust tuples, where each member implements `IntoPyObject`,
- `Bound<'_, PyTuple>`
- `Py<PyTuple>`
Rust tuples may make use of [`vectorcall`] where as `Bound<'_, PyTuple>` and `Py<PyTuple>` can only use [`tp_call`].
For maximum performance prefer using Rust tuples as arguments.
[`tp_call`]: https://docs.python.org/3/c-api/call.html#the-tp-call-protocol
[`vectorcall`]: https://docs.python.org/3/c-api/call.html#the-vectorcall-protocol
## Detach from the interpreter for long-running Rust-only work
When executing Rust code which does not need to interact with the Python interpreter, use [`Python::detach`] to allow the Python interpreter to proceed without waiting for the current thread.
On the GIL-enabled build, this is crucial for best performance as only a single thread may ever be attached at a time.
On the free-threaded build, this is still best practice as there are several "stop the world" events (such as garbage collection) where all threads attached to the Python interpreter are forced to wait.
As a rule of thumb, attaching and detaching from the Python interpreter takes less than a millisecond, so any work which is expected to take multiple milliseconds can likely benefit from detaching from the interpreter.
[`Python::detach`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.detach
## Disable the global reference pool
PyO3 uses global mutable state to keep track of deferred reference count updates implied by `impl<T> Drop for Py<T>` being called without being attached to the interpreter.
The necessary synchronization to obtain and apply these reference count updates when PyO3-based code next attaches to the interpreter is somewhat expensive and can become a significant part of the cost of crossing the Python-Rust boundary.
This functionality can be avoided by setting the `pyo3_disable_reference_pool` conditional compilation flag.
This removes the global reference pool and the associated costs completely.
However, it does _not_ remove the `Drop` implementation for `Py<T>` which is necessary to interoperate with existing Rust code written without PyO3-based code in mind.
To stay compatible with the wider Rust ecosystem in these cases, we keep the implementation but abort when `Drop` is called without being attached to the interpreter.
If `pyo3_leak_on_drop_without_reference_pool` is additionally enabled, objects dropped without being attached to Python will be leaked instead which is always sound but might have determinal effects like resource exhaustion in the long term.
This limitation is important to keep in mind when this setting is used, especially when embedding Python code into a Rust application as it is quite easy to accidentally drop a `Py<T>` (or types containing it like `PyErr`, `PyBackedStr` or `PyBackedBytes`) returned from `Python::attach` without making sure to re-attach beforehand.
For example, the following code
```rust,ignore
# use pyo3::prelude::*;
# use pyo3::types::PyList;
let numbers: Py<PyList> = Python::attach(|py| PyList::empty(py).unbind());
Python::attach(|py| {
numbers.bind(py).append(23).unwrap();
});
Python::attach(|py| {
numbers.bind(py).append(42).unwrap();
});
```
will abort if the list not explicitly disposed via
```rust
# use pyo3::prelude::*;
# use pyo3::types::PyList;
let numbers: Py<PyList> = Python::attach(|py| PyList::empty(py).unbind());
Python::attach(|py| {
numbers.bind(py).append(23).unwrap();
});
Python::attach(|py| {
numbers.bind(py).append(42).unwrap();
});
Python::attach(move |py| {
drop(numbers);
});
```
+63
View File
@@ -0,0 +1,63 @@
# Calling Python in Rust code
This chapter of the guide documents some ways to interact with Python code from Rust.
Below is an introduction to the `'py` lifetime and some general remarks about how PyO3's API reasons about Python code.
The subchapters also cover the following topics:
- Python object types available in PyO3's API
- How to work with Python exceptions
- How to call Python functions
- How to execute existing Python code
## The `'py` lifetime
To safely interact with the Python interpreter a Rust thread must be [attached] to the Python interpreter.
PyO3 has a `Python<'py>` token that is used to prove that these conditions are met.
Its lifetime `'py` is a central part of PyO3's API.
The `Python<'py>` token serves three purposes:
- It provides global APIs for the Python interpreter, such as [`py.eval()`][eval] and [`py.import()`][import].
- It can be passed to functions that require a proof of attachment, such as [`Py::clone_ref`][clone_ref].
- Its lifetime `'py` is used to bind many of PyO3's types to the Python interpreter, such as [`Bound<'py, T>`][Bound].
PyO3's types that are bound to the `'py` lifetime, for example `Bound<'py, T>`, all contain a `Python<'py>` token.
This means they have full access to the Python interpreter and offer a complete API for interacting with Python objects.
Consult [PyO3's API documentation][obtaining-py] to learn how to acquire one of these tokens.
### The Global Interpreter Lock
Prior to the introduction of free-threaded Python (first available in 3.13, fully supported in 3.14), the Python interpreter was made thread-safe by the global interpreter lock.
This ensured that only one Python thread can use the Python interpreter and its API at the same time.
Historically, Rust code was able to use the GIL as a synchronization guarantee, but the introduction of free-threaded Python removed this possibility.
The [`pyo3::sync`] module offers synchronization tools which abstract over both Python builds.
To enable any parallelism on the GIL-enabled build, and best throughput on the free-threaded build, non-Python operations (system calls and native Rust code) should consider detaching from the Python interpreter to allow other work to proceed.
See [the section on parallelism](parallelism.md) for how to do that using PyO3's API.
## Python's memory model
Python's memory model differs from Rust's memory model in two key ways:
- There is no concept of ownership; all Python objects are shared and usually implemented via reference counting
- There is no concept of exclusive (`&mut`) references; any reference can mutate a Python object
PyO3's API reflects this by providing smart pointer types, `Py<T>`, `Bound<'py, T>`, and (the very rarely used) `Borrowed<'a, 'py, T>`.
These smart pointers all use Python reference counting.
See the [subchapter on types](./types.md) for more detail on these types.
Because of the lack of exclusive `&mut` references, PyO3's APIs for Python objects, for example [`PyListMethods::append`], use shared references.
This is safe because Python objects have internal mechanisms to prevent data races (as of time of writing, the Python GIL).
[attached]: https://docs.python.org/3.14/glossary.html#term-attached-thread-state
[obtaining-py]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#obtaining-a-python-token
[`pyo3::sync`]: {{#PYO3_DOCS_URL}}/pyo3/sync/index.html
[eval]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval
[import]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.import
[clone_ref]: {{#PYO3_DOCS_URL}}/pyo3/prelude/struct.Py.html#method.clone_ref
[Bound]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html
[`PyListMethods::append`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyListMethods.html#tymethod.append
@@ -0,0 +1,391 @@
# Executing existing Python code
If you already have some existing Python code that you need to execute from Rust, the following FAQs can help you select the right PyO3 functionality for your situation:
## Want to access Python APIs? Then use `PyModule::import`
[`PyModule::import`] can be used to get handle to a Python module from Rust.
You can use this to import and use any Python module available in your environment.
```rust
use pyo3::prelude::*;
fn main() -> PyResult<()> {
Python::attach(|py| {
let builtins = PyModule::import(py, "builtins")?;
let total: i32 = builtins
.getattr("sum")?
.call1((vec![1, 2, 3],))?
.extract()?;
assert_eq!(total, 6);
Ok(())
})
}
```
[`PyModule::import`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.import
## Want to run just an expression? Then use `eval`
[`Python::eval`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval) is a method to execute a [Python expression](https://docs.python.org/3/reference/expressions.html) and return the evaluated value as a `Bound<'py, PyAny>` object.
```rust
use pyo3::prelude::*;
# fn main() -> Result<(), ()> {
Python::attach(|py| {
let result = py
.eval(c"[i * 10 for i in range(5)]", None, None)
.map_err(|e| {
e.print_and_set_sys_last_vars(py);
})?;
let res: Vec<i64> = result.extract().unwrap();
assert_eq!(res, vec![0, 10, 20, 30, 40]);
Ok(())
})
# }
```
## Want to run statements? Then use `run`
[`Python::run`] is a method to execute one or more [Python statements](https://docs.python.org/3/reference/simple_stmts.html).
This method returns nothing (like any Python statement), but you can get access to manipulated objects via the `locals` dict.
You can also use the [`py_run!`] macro, which is a shorthand for [`Python::run`].
Since [`py_run!`] panics on exceptions, we recommend you use this macro only for quickly testing your Python extension modules.
```rust
use pyo3::prelude::*;
use pyo3::py_run;
# fn main() {
#[pyclass]
struct UserData {
id: u32,
name: String,
}
#[pymethods]
impl UserData {
fn as_tuple(&self) -> (u32, String) {
(self.id, self.name.clone())
}
fn __repr__(&self) -> PyResult<String> {
Ok(format!("User {}(id: {})", self.name, self.id))
}
}
Python::attach(|py| {
let userdata = UserData {
id: 34,
name: "Yu".to_string(),
};
let userdata = Py::new(py, userdata).unwrap();
let userdata_as_tuple = (34, "Yu");
py_run!(py, userdata userdata_as_tuple, r#"
assert repr(userdata) == "User Yu(id: 34)"
assert userdata.as_tuple() == userdata_as_tuple
"#);
})
# }
```
## You have a Python file or code snippet? Then use `PyModule::from_code`
[`PyModule::from_code`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.from_code) can be used to generate a Python module which can then be used just as if it was imported with `PyModule::import`.
**Warning**: This will compile and execute code.
**Never** pass untrusted code to this function!
```rust
use pyo3::{prelude::*, types::IntoPyDict};
# fn main() -> PyResult<()> {
Python::attach(|py| {
let activators = PyModule::from_code(
py,
cr#"
def relu(x):
"""see https://en.wikipedia.org/wiki/Rectifier_(neural_networks)"""
return max(0.0, x)
def leaky_relu(x, slope=0.01):
return x if x >= 0 else x * slope
"#,
c"activators.py",
c"activators",
)?;
let relu_result: f64 = activators.getattr("relu")?.call1((-1.0,))?.extract()?;
assert_eq!(relu_result, 0.0);
let kwargs = [("slope", 0.2)].into_py_dict(py)?;
let lrelu_result: f64 = activators
.getattr("leaky_relu")?
.call((-1.0,), Some(&kwargs))?
.extract()?;
assert_eq!(lrelu_result, -0.2);
# Ok(())
})
# }
```
## Want to embed Python in Rust with additional modules?
Python maintains the `sys.modules` dict as a cache of all imported modules.
An import in Python will first attempt to lookup the module from this dict, and if not present will use various strategies to attempt to locate and load the module.
The [`append_to_inittab`]({{#PYO3_DOCS_URL}}/pyo3/macro.append_to_inittab.html) macro can be used to add additional `#[pymodule]` modules to an embedded Python interpreter.
The macro **must** be invoked _before_ initializing Python.
As an example, the below adds the module `foo` to the embedded interpreter:
```rust
use pyo3::prelude::*;
#[pymodule]
mod foo {
use pyo3::prelude::*;
#[pyfunction]
fn add_one(x: i64) -> i64 {
x + 1
}
}
# #[cfg(not(all(Py_LIMITED_API, Py_GIL_DISABLED)))]
fn main() -> PyResult<()> {
pyo3::append_to_inittab!(foo);
Python::attach(|py| Python::run(py, c"import foo; foo.add_one(6)", None, None))
}
# #[cfg(all(Py_LIMITED_API, Py_GIL_DISABLED))]
# fn main() -> () {}
```
If `append_to_inittab` cannot be used due to constraints in the program, an alternative is to create a module using [`PyModule::new`] and insert it manually into `sys.modules`:
```rust
use pyo3::prelude::*;
use pyo3::types::PyDict;
#[pyfunction]
pub fn add_one(x: i64) -> i64 {
x + 1
}
fn main() -> PyResult<()> {
Python::attach(|py| {
// Create new module
let foo_module = PyModule::new(py, "foo")?;
foo_module.add_function(wrap_pyfunction!(add_one, &foo_module)?)?;
// Import and get sys.modules
let sys = PyModule::import(py, "sys")?;
let py_modules: Bound<'_, PyDict> = sys.getattr("modules")?.cast_into()?;
// Insert foo into sys.modules
py_modules.set_item("foo", foo_module)?;
// Now we can import + run our python code
Python::run(py, c"import foo; foo.add_one(6)", None, None)
})
}
```
## Include multiple Python files
You can include a file at compile time by using [`std::include_str`](https://doc.rust-lang.org/std/macro.include_str.html) macro.
Or you can load a file at runtime by using [`std::fs::read_to_string`](https://doc.rust-lang.org/std/fs/fn.read_to_string.html) function.
Many Python files can be included and loaded as modules.
If one file depends on another you must preserve correct order while declaring `PyModule`.
Example directory structure:
```text
.
├── Cargo.lock
├── Cargo.toml
├── python_app
│ ├── app.py
│ └── utils
│ └── foo.py
└── src
└── main.rs
```
`python_app/app.py`:
```python
from utils.foo import bar
def run():
return bar()
```
`python_app/utils/foo.py`:
```python
def bar():
return "baz"
```
The example below shows:
- how to include content of `app.py` and `utils/foo.py` into your rust binary
- how to call function `run()` (declared in `app.py`) that needs function
imported from `utils/foo.py`
`src/main.rs`:
```rust,ignore
use pyo3::prelude::*;
use pyo3_ffi::c_str;
fn main() -> PyResult<()> {
let py_foo = c_str!(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/python_app/utils/foo.py"
)));
let py_app = c_str!(include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/python_app/app.py")));
let from_python = Python::attach(|py| -> PyResult<Py<PyAny>> {
PyModule::from_code(py, py_foo, c"foo.py", c"utils.foo")?;
let app: Py<PyAny> = PyModule::from_code(py, py_app, c"app.py", c"")?
.getattr("run")?
.into();
app.call0(py)
});
println!("py: {}", from_python?);
Ok(())
}
```
The example below shows:
- how to load content of `app.py` at runtime so that it sees its dependencies
automatically
- how to call function `run()` (declared in `app.py`) that needs function
imported from `utils/foo.py`
It is recommended to use absolute paths because then your binary can be run from anywhere as long as your `app.py` is in the expected directory (in this example that directory is `/usr/share/python_app`).
`src/main.rs`:
```rust,no_run
use pyo3::prelude::*;
use pyo3::types::PyList;
use std::fs;
use std::path::Path;
use std::ffi::CString;
fn main() -> PyResult<()> {
let path = Path::new("/usr/share/python_app");
let py_app = CString::new(fs::read_to_string(path.join("app.py"))?)?;
let from_python = Python::attach(|py| -> PyResult<Py<PyAny>> {
let syspath = py
.import("sys")?
.getattr("path")?
.cast_into::<PyList>()?;
syspath.insert(0, path)?;
let app: Py<PyAny> = PyModule::from_code(py, py_app.as_c_str(), c"app.py", c"")?
.getattr("run")?
.into();
app.call0(py)
});
println!("py: {}", from_python?);
Ok(())
}
```
## Need to use a context manager from Rust?
Use context managers by directly invoking `__enter__` and `__exit__`.
```rust
use pyo3::prelude::*;
fn main() {
Python::attach(|py| {
let custom_manager = PyModule::from_code(
py,
cr#"
class House(object):
def __init__(self, address):
self.address = address
def __enter__(self):
print(f"Welcome to {self.address}!")
def __exit__(self, type, value, traceback):
if type:
print(f"Sorry you had {type} trouble at {self.address}")
else:
print(f"Thank you for visiting {self.address}, come again soon!")
"#,
c"house.py",
c"house",
)
.unwrap();
let house_class = custom_manager.getattr("House").unwrap();
let house = house_class.call1(("123 Main Street",)).unwrap();
house.call_method0("__enter__").unwrap();
let result = py.eval(c"undefined_variable + 1", None, None);
// If the eval threw an exception we'll pass it through to the context manager.
// Otherwise, __exit__ is called with empty arguments (Python "None").
match result {
Ok(_) => {
let none = py.None();
house
.call_method1("__exit__", (&none, &none, &none))
.unwrap();
}
Err(e) => {
house
.call_method1(
"__exit__",
(
e.get_type(py),
e.value(py),
e.traceback(py),
),
)
.unwrap();
}
}
})
}
```
## Handling system signals/interrupts (Ctrl-C)
The best way to handle system signals when running Rust code is to periodically call `Python::check_signals` to handle any signals captured by Python's signal handler.
See also [the FAQ entry](../faq.md#ctrl-c-doesnt-do-anything-while-my-rust-code-is-executing).
Alternatively, set Python's `signal` module to take the default action for a signal:
```rust
use pyo3::prelude::*;
# fn main() -> PyResult<()> {
Python::attach(|py| -> PyResult<()> {
let signal = py.import("signal")?;
// Set SIGINT to have the default action
signal
.getattr("signal")?
.call1((signal.getattr("SIGINT")?, signal.getattr("SIG_DFL")?))?;
Ok(())
})
# }
```
[`py_run!`]: {{#PYO3_DOCS_URL}}/pyo3/macro.py_run.html
[`Python::run`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.run
[`PyModule::new`]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyModule.html#method.new
+115
View File
@@ -0,0 +1,115 @@
# Calling Python functions
The `Bound<'py, T>` smart pointer (such as `Bound<'py, PyAny>`, `Bound<'py, PyList>`, or `Bound<'py, MyClass>`) can be used to call Python functions.
PyO3 offers two APIs to make function calls:
- [`call`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call) - call any callable Python object.
- [`call_method`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method) - call a method on the Python object.
Both of these APIs take `args` and `kwargs` arguments (for positional and keyword arguments respectively).
There are variants for less complex calls:
- [`call1`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call1) and [`call_method1`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method1) to call only with positional `args`.
- [`call0`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call0) and [`call_method0`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.call_method0) to call with no arguments.
For convenience the [`Py<T>`](../types.md#pyt) smart pointer also exposes these same six API methods, but needs a `Python` token as an additional first argument to prove the thread is attached to the Python interpreter.
The example below calls a Python function behind a `Py<PyAny>` reference:
```rust
use pyo3::prelude::*;
use pyo3::types::PyTuple;
fn main() -> PyResult<()> {
let arg1 = "arg1";
let arg2 = "arg2";
let arg3 = "arg3";
Python::attach(|py| {
let fun: Py<PyAny> = PyModule::from_code(
py,
c"def example(*args, **kwargs):
if args != ():
print('called with args', args)
if kwargs != {}:
print('called with kwargs', kwargs)
if args == () and kwargs == {}:
print('called with no arguments')",
c"example.py",
c"",
)?
.getattr("example")?
.into();
// call object without any arguments
fun.call0(py)?;
// pass object with Rust tuple of positional arguments
let args = (arg1, arg2, arg3);
fun.call1(py, args)?;
// call object with Python tuple of positional arguments
let args = PyTuple::new(py, &[arg1, arg2, arg3])?;
fun.call1(py, args)?;
Ok(())
})
}
```
## Creating keyword arguments
For the `call` and `call_method` APIs, `kwargs` are `Option<&Bound<'py, PyDict>>`, so can either be `None` or `Some(&dict)`.
You can use the [`IntoPyDict`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.IntoPyDict.html) trait to convert other dict-like containers, e.g. `HashMap` or `BTreeMap`, as well as tuples with up to 10 elements and `Vec`s where each element is a two-element tuple.
To pass keyword arguments of different types, construct a `PyDict` object.
```rust
use pyo3::prelude::*;
use pyo3::types::{PyDict, IntoPyDict};
use std::collections::HashMap;
fn main() -> PyResult<()> {
let key1 = "key1";
let val1 = 1;
let key2 = "key2";
let val2 = 2;
Python::attach(|py| {
let fun: Py<PyAny> = PyModule::from_code(
py,
c"def example(*args, **kwargs):
if args != ():
print('called with args', args)
if kwargs != {}:
print('called with kwargs', kwargs)
if args == () and kwargs == {}:
print('called with no arguments')",
c"example.py",
c"",
)?
.getattr("example")?
.into();
// call object with PyDict
let kwargs = [(key1, val1)].into_py_dict(py)?;
fun.call(py, (), Some(&kwargs))?;
// pass arguments as Vec
let kwargs = vec![(key1, val1), (key2, val2)];
fun.call(py, (), Some(&kwargs.into_py_dict(py)?))?;
// pass arguments as HashMap
let mut kwargs = HashMap::<&str, i32>::new();
kwargs.insert(key1, 1);
fun.call(py, (), Some(&kwargs.into_py_dict(py)?))?;
// pass arguments of different types as PyDict
let kwargs = PyDict::new(py);
kwargs.set_item(key1, val1)?;
kwargs.set_item(key2, "string")?;
fun.call(py, (), Some(&kwargs))?;
Ok(())
})
}
```
+256
View File
@@ -0,0 +1,256 @@
# Typing and IDE hints for your Python package
PyO3 provides an easy to use interface to code native Python libraries in Rust.
The accompanying Maturin allows you to build and publish them as a package.
Yet, for a better user experience, Python libraries should provide typing hints and documentation for all public entities, so that IDEs can show them during development and type analyzing tools such as `mypy` can use them to properly verify the code.
Currently the best solution for the problem is to manually maintain `*.pyi` files and ship them along with the package.
PyO3 is working on automated their generation.
See the [type stub generation](type-stub.md) documentation for a description of the current state of automated generation.
## Introduction to `pyi` files
`pyi` files (an abbreviation for `Python Interface`) are called "stub files" in most of the documentation related to them.
A very good definition of what it is can be found in [old MyPy documentation](https://github.com/python/mypy/wiki/Creating-Stubs-For-Python-Modules):
> A stubs file only contains a description of the public interface of the module without any implementations.
There is also [extensive documentation on type stubs on the official Python typing documentation](https://typing.readthedocs.io/en/latest/source/stubs.html).
Most Python developers probably already encountered them when trying to use their IDE's "Go to Definition" function on any builtin type.
For example, the definitions of a few standard exceptions look like this:
```python
class BaseException(object):
args: Tuple[Any, ...]
__cause__: BaseException | None
__context__: BaseException | None
__suppress_context__: bool
__traceback__: TracebackType | None
def __init__(self, *args: object) -> None: ...
def __str__(self) -> str: ...
def __repr__(self) -> str: ...
def with_traceback(self: _TBE, tb: TracebackType | None) -> _TBE: ...
class SystemExit(BaseException):
code: int
class Exception(BaseException): ...
class StopIteration(Exception):
value: Any
```
As we can see, those are not full definitions containing implementation, but just a description of the interface.
It is usually all that the user of the library needs.
### What do the PEPs say?
At the time of writing this documentation, the `pyi` files are referenced in four PEPs.
[PEP8 - Style Guide for Python Code - #Function Annotations](https://www.python.org/dev/peps/pep-0008/#function-annotations) (last point) recommends all third party library creators to provide stub files as the source of knowledge about the package for type checker tools.
> (...) it is expected that users of third party library packages may want to run type checkers over those packages. For this purpose [PEP 484](https://www.python.org/dev/peps/pep-0484) recommends the use of stub files: .pyi files that are read by the type checker in preference of the corresponding .py files. (...)
[PEP484 - Type Hints - #Stub Files](https://www.python.org/dev/peps/pep-0484/#stub-files) defines stub files as follows.
> Stub files are files containing type hints that are only for use by the type checker, not at runtime.
It contains a specification for them (highly recommended reading, since it contains at least one thing that is not used in normal Python code) and also some general information about where to store the stub files.
[PEP561 - Distributing and Packaging Type Information](https://www.python.org/dev/peps/pep-0561/) describes in detail how to build packages that will enable type checking.
In particular it contains information about how the stub files must be distributed in order for type checkers to use them.
[PEP560 - Core support for typing module and generic types](https://www.python.org/dev/peps/pep-0560/) describes the details on how Python's type system internally supports generics, including both runtime behavior and integration with static type checkers.
## How to do it?
[PEP561](https://www.python.org/dev/peps/pep-0561/) recognizes three ways of distributing type information:
- `inline` - the typing is placed directly in source (`py`) files;
- `separate package with stub files` - the typing is placed in `pyi` files distributed in their own, separate package;
- `in-package stub files` - the typing is placed in `pyi` files distributed in the same package as source files.
The first way is tricky with PyO3 since we do not have `py` files.
When it has been investigated and necessary changes are implemented, this document will be updated.
The second way is easy to do, and the whole work can be fully separated from the main library code.
The example repo for the package with stub files can be found in [PEP561 references section](https://www.python.org/dev/peps/pep-0561/#references): [Stub package repository](https://github.com/ethanhs/stub-package)
The third way is described below.
### Including `pyi` files in your PyO3/Maturin build package
When source files are in the same package as stub files, they should be placed next to each other.
We need a way to do that with Maturin.
Also, in order to mark our package as typing-enabled we need to add an empty file named `py.typed` to the package.
#### If you do not have other Python files
If you do not need to add any other Python files apart from `pyi` to the package, Maturin provides a way to do most of the work for you.
As documented in the [Maturin Guide](https://github.com/PyO3/maturin/#mixed-rustpython-projects), the only thing you need to do is to create a stub file for your module named `<module_name>.pyi` in your project root and Maturin will do the rest.
```text
my-rust-project/
├── Cargo.toml
├── my_project.pyi # <<< add type stubs for Rust functions in the my_project module here
├── pyproject.toml
└── src
└── lib.rs
```
For an example `pyi` file see the [`my_project.pyi` content](#my_projectpyi-content) section.
#### If you need other Python files
If you need to add other Python files apart from `pyi` to the package, you can do it also, but that requires some more work.
Maturin provides an easy way to add files to a package ([documentation](https://github.com/PyO3/maturin/blob/0dee40510083c03607834c821eea76964140a126/Readme.md#mixed-rustpython-projects)).
You just need to create a folder with the name of your module next to the `Cargo.toml` file (for customization see documentation linked above).
The folder structure would be:
```text
my-project
├── Cargo.toml
├── my_project
│ ├── __init__.py
│ ├── my_project.pyi
│ ├── other_python_file.py
│ └── py.typed
├── pyproject.toml
├── Readme.md
└── src
└── lib.rs
```
Let's go a little bit more into detail regarding the files inside the package folder.
##### `__init__.py` content
As we now specify our own package content, we have to provide the `__init__.py` file, so the folder is treated as a package and we can import things from it.
We can always use the same content that Maturin creates for us if we do not specify a Python source folder.
For PyO3 bindings it would be:
```python
from .my_project import *
```
That way everything that is exposed by our extension module can be imported directly from the package.
##### `py.typed` requirement
As stated in [PEP561](https://www.python.org/dev/peps/pep-0561/):
> Package maintainers who wish to support type checking of their code MUST add a marker file named py.typed to their package supporting typing. This marker applies recursively: if a top-level package includes it, all its sub-packages MUST support type checking as well.
If we do not include that file, some IDEs might still use our `pyi` files to show hints, but the type checkers might not.
MyPy will raise an error in this situation:
```text
error: Skipping analyzing "my_project": found module but no type hints or library stubs
```
The file is just a marker file, so it should be empty.
##### `my_project.pyi` content
Our module stub file.
This document does not aim at describing how to write them, since you can find a lot of documentation on it, starting from the already quoted [PEP484](https://www.python.org/dev/peps/pep-0484/#stub-files).
The example can look like this:
```python
class Car:
"""
A class representing a car.
:param body_type: the name of body type, e.g. hatchback, sedan
:param horsepower: power of the engine in horsepower
"""
def __init__(self, body_type: str, horsepower: int) -> None: ...
@classmethod
def from_unique_name(cls, name: str) -> 'Car':
"""
Creates a Car based on unique name
:param name: model name of a car to be created
:return: a Car instance with default data
"""
def best_color(self) -> str:
"""
Gets the best color for the car.
:return: the name of the color our great algorithm thinks is the best for this car
"""
```
### Supporting Generics
Type annotations can also be made generic in Python.
They are useful for working with different types while maintaining type safety.
Usually, generic classes inherit from the `typing.Generic` metaclass.
<!-- no-glossary-link:wheel -->
Take for example the following `.pyi` file that specifies a `Car` that can accept multiple types of wheels:
```python
from typing import Generic, TypeVar
W = TypeVar('W')
class Car(Generic[W]):
def __init__(self, wheels: list[W]) -> None: ...
def get_wheels(self) -> list[W]: ...
def change_wheel(self, wheel_number: int, wheel: W) -> None: ...
```
This way, the end-user can specify the type with variables such as `truck: Car[SteelWheel] = ...` and `f1_car: Car[AlloyWheel] = ...`.
There is also a special syntax for specifying generic types in Python 3.12+:
```python
class Car[W]:
def __init__(self, wheels: list[W]) -> None: ...
def get_wheels(self) -> list[W]: ...
```
#### Runtime Behaviour
Stub files (`pyi`) are only useful for static type checkers and ignored at runtime.
Therefore, PyO3 classes do not inherit from `typing.Generic` even if specified in the stub files.
This can cause some runtime issues, as annotating a variable like `f1_car: Car[AlloyWheel] = ...` can make Python call magic methods that are not defined.
To overcome this limitation, implementers can pass the `generic` parameter to `pyclass` in Rust:
```rust ignore
#[pyclass(generic)]
```
#### Advanced Users
`#[pyclass(generic)]` implements a very simple runtime behavior that accepts any generic argument.
Advanced users can opt to manually implement [`__class_getitem__`](https://docs.python.org/3/reference/datamodel.html#emulating-generic-types) for the generic class to have more control.
```rust ignore
impl MyClass {
#[classmethod]
#[pyo3(signature = (key, /))]
pub fn __class_getitem__(
cls: &Bound<'_, PyType>,
key: &Bound<'_, PyAny>,
) -> PyResult<Py<PyAny>> {
/* implementation details */
}
}
```
Note that [`pyo3::types::PyGenericAlias`][pygenericalias] can be helpful when implementing `__class_getitem__` as it can create [`types.GenericAlias`][genericalias] objects from Rust.
[pygenericalias]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyGenericAlias.html
[genericalias]: https://docs.python.org/3/library/types.html#types.GenericAlias
+13
View File
@@ -0,0 +1,13 @@
# Using Rust from Python
This chapter of the guide is dedicated to explaining how to wrap Rust code into Python objects.
PyO3 uses Rust's "procedural macros" to provide a powerful yet simple API to denote what Rust code should map into Python objects.
PyO3 can create three types of Python objects:
- Python modules, via the `#[pymodule]` macro
- Python functions, via the `#[pyfunction]` macro
- Python classes, via the `#[pyclass]` macro (plus `#[pymethods]` to define methods for those classes)
The following subchapters go through each of these in turn.
+561
View File
@@ -0,0 +1,561 @@
# Using in Python a Rust function with trait bounds
PyO3 allows for easy conversion from Rust to Python for certain functions and classes (see the [conversion table](conversions/tables.md)).
However, it is not always straightforward to convert Rust code that requires a given trait implementation as an argument.
This tutorial explains how to convert a Rust function that takes a trait as argument for use in Python with classes implementing the same methods as the trait.
Why is this useful?
## Pros
- Make your Rust code available to Python users
- Code complex algorithms in Rust with the help of the borrow checker
### Cons
- Not as fast as native Rust (type conversion has to be performed and one part of the code runs in Python)
- You need to adapt your code to expose it
## Example
Let's work with the following basic example of an implementation of a optimization solver operating on a given model.
Let's say we have a function `solve` that operates on a model and mutates its state.
The argument of the function can be any model that implements the `Model` trait :
```rust,no_run
# #![allow(dead_code)]
pub trait Model {
fn set_variables(&mut self, inputs: &Vec<f64>);
fn compute(&mut self);
fn get_results(&self) -> Vec<f64>;
}
pub fn solve<T: Model>(model: &mut T) {
println!("Magic solver that mutates the model into a resolved state");
}
```
Let's assume we have the following constraints:
- We cannot change that code as it runs on many Rust models.
- We also have many Python models that cannot be solved as this solver is not available in that language.
Rewriting it in Python would be cumbersome and error-prone, as everything is already available in Rust.
How could we expose this solver to Python thanks to PyO3 ?
## Implementation of the trait bounds for the Python class
If a Python class implements the same three methods as the `Model` trait, it seems logical it could be adapted to use the solver.
However, it is not possible to pass a `Py<PyAny>` to it as it does not implement the Rust trait (even if the Python model has the required methods).
In order to implement the trait, we must write a wrapper around the calls in Rust to the Python model.
The method signatures must be the same as the trait, keeping in mind that the Rust trait cannot be changed for the purpose of making the code available in Python.
The Python model we want to expose is the following one, which already contains all the required methods:
```python
class Model:
def set_variables(self, inputs):
self.inputs = inputs
def compute(self):
self.results = [elt**2 - 3 for elt in self.inputs]
def get_results(self):
return self.results
```
The following wrapper will call the Python model from Rust, using a struct to hold the model as a `PyAny` object:
```rust,no_run
# #![allow(dead_code)]
use pyo3::prelude::*;
use pyo3::types::PyList;
# pub trait Model {
# fn set_variables(&mut self, inputs: &Vec<f64>);
# fn compute(&mut self);
# fn get_results(&self) -> Vec<f64>;
# }
struct UserModel {
model: Py<PyAny>,
}
impl Model for UserModel {
fn set_variables(&mut self, var: &Vec<f64>) {
println!("Rust calling Python to set the variables");
Python::attach(|py| {
self.model
.bind(py)
.call_method("set_variables", (PyList::new(py, var).unwrap(),), None)
.unwrap();
})
}
fn get_results(&self) -> Vec<f64> {
println!("Rust calling Python to get the results");
Python::attach(|py| {
self.model
.bind(py)
.call_method("get_results", (), None)
.unwrap()
.extract()
.unwrap()
})
}
fn compute(&mut self) {
println!("Rust calling Python to perform the computation");
Python::attach(|py| {
self.model
.bind(py)
.call_method("compute", (), None)
.unwrap();
})
}
}
```
Now that this bit is implemented, let's expose the model wrapper to Python.
Let's add the PyO3 annotations and add a constructor:
```rust,no_run
# #![allow(dead_code)]
# fn main() {}
# pub trait Model {
# fn set_variables(&mut self, inputs: &Vec<f64>);
# fn compute(&mut self);
# fn get_results(&self) -> Vec<f64>;
# }
# use pyo3::prelude::*;
#[pyclass]
struct UserModel {
model: Py<PyAny>,
}
#[pymethods]
impl UserModel {
#[new]
pub fn new(model: Py<PyAny>) -> Self {
UserModel { model }
}
}
#[pymodule]
mod trait_exposure {
#[pymodule_export]
use super::UserModel;
}
```
Now we add the PyO3 annotations to the trait implementation:
```rust,ignore
#[pymethods]
impl Model for UserModel {
// the previous trait implementation
}
```
However, the previous code will not compile.
The compilation error is the following one: `error: #[pymethods] cannot be used on trait impl blocks`
That's a bummer!
However, we can write a second wrapper around these functions to call them directly.
This wrapper will also perform the type conversions between Python and Rust.
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
# use pyo3::types::PyList;
#
# pub trait Model {
# fn set_variables(&mut self, inputs: &Vec<f64>);
# fn compute(&mut self);
# fn get_results(&self) -> Vec<f64>;
# }
#
# #[pyclass]
# struct UserModel {
# model: Py<PyAny>,
# }
#
# impl Model for UserModel {
# fn set_variables(&mut self, var: &Vec<f64>) {
# println!("Rust calling Python to set the variables");
# Python::attach(|py| {
# self.model.bind(py)
# .call_method("set_variables", (PyList::new(py, var).unwrap(),), None)
# .unwrap();
# })
# }
#
# fn get_results(&self) -> Vec<f64> {
# println!("Rust calling Python to get the results");
# Python::attach(|py| {
# self.model
# .bind(py)
# .call_method("get_results", (), None)
# .unwrap()
# .extract()
# .unwrap()
# })
# }
#
# fn compute(&mut self) {
# println!("Rust calling Python to perform the computation");
# Python::attach(|py| {
# self.model
# .bind(py)
# .call_method("compute", (), None)
# .unwrap();
# })
#
# }
# }
#[pymethods]
impl UserModel {
pub fn set_variables(&mut self, var: Vec<f64>) {
println!("Set variables from Python calling Rust");
Model::set_variables(self, &var)
}
pub fn get_results(&mut self) -> Vec<f64> {
println!("Get results from Python calling Rust");
Model::get_results(self)
}
pub fn compute(&mut self) {
println!("Compute from Python calling Rust");
Model::compute(self)
}
}
```
This wrapper handles the type conversion between the PyO3 requirements and the trait.
In order to meet PyO3 requirements, this wrapper must:
- return an object of type `PyResult`
- use only values, not references in the method signatures
Let's run the file python file:
```python
class Model:
def set_variables(self, inputs):
self.inputs = inputs
def compute(self):
self.results = [elt**2 - 3 for elt in self.inputs]
def get_results(self):
return self.results
if __name__=="__main__":
import trait_exposure
myModel = Model()
my_rust_model = trait_exposure.UserModel(myModel)
my_rust_model.set_variables([2.0])
print("Print value from Python: ", myModel.inputs)
my_rust_model.compute()
print("Print value from Python through Rust: ", my_rust_model.get_results())
print("Print value directly from Python: ", myModel.get_results())
```
This outputs:
```block
Set variables from Python calling Rust
Set variables from Rust calling Python
Print value from Python: [2.0]
Compute from Python calling Rust
Compute from Rust calling Python
Get results from Python calling Rust
Get results from Rust calling Python
Print value from Python through Rust: [1.0]
Print value directly from Python: [1.0]
```
We have now successfully exposed a Rust model that implements the `Model` trait to Python!
We will now expose the `solve` function, but before, let's talk about types errors.
## Type errors in Python
What happens if you have type errors when using Python and how can you improve the error messages?
### Wrong types in Python function arguments
Let's assume in the first case that you will use in your Python file `my_rust_model.set_variables(2.0)` instead of `my_rust_model.set_variables([2.0])`.
The Rust signature expects a vector, which corresponds to a list in Python.
What happens if instead of a vector, we pass a single value ?
At the execution of Python, we get :
```block
File "main.py", line 15, in <module>
my_rust_model.set_variables(2)
TypeError
```
It is a type error and Python points to it, so it's easy to identify and solve.
### Wrong types in Python method signatures
Let's assume now that the return type of one of the methods of our Model class is wrong, for example the `get_results` method that is expected to return a `Vec<f64>` in Rust, a list in Python.
```python
class Model:
def set_variables(self, inputs):
self.inputs = inputs
def compute(self):
self.results = [elt**2 -3 for elt in self.inputs]
def get_results(self):
return self.results[0]
#return self.results <-- this is the expected output
```
This call results in the following panic:
```block
pyo3_runtime.PanicException: called `Result::unwrap()` on an `Err` value: PyErr { type: Py(0x10dcf79f0, PhantomData) }
```
This error code is not helpful for a Python user that does not know anything about Rust, or someone that does not know PyO3 was used to interface the Rust code.
However, as we are responsible for making the Rust code available to Python, we can do something about it.
The issue is that we called `unwrap` anywhere we could, and therefore any panic from PyO3 will be directly forwarded to the end user.
Let's modify the code performing the type conversion to give a helpful error message to the Python user:
We used in our `get_results` method the following call that performs the type conversion:
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
# use pyo3::types::PyList;
#
# pub trait Model {
# fn set_variables(&mut self, inputs: &Vec<f64>);
# fn compute(&mut self);
# fn get_results(&self) -> Vec<f64>;
# }
#
# #[pyclass]
# struct UserModel {
# model: Py<PyAny>,
# }
impl Model for UserModel {
fn get_results(&self) -> Vec<f64> {
println!("Rust calling Python to get the results");
Python::attach(|py| {
self.model
.bind(py)
.call_method("get_results", (), None)
.unwrap()
.extract()
.unwrap()
})
}
# fn set_variables(&mut self, var: &Vec<f64>) {
# println!("Rust calling Python to set the variables");
# Python::attach(|py| {
# self.model.bind(py)
# .call_method("set_variables", (PyList::new(py, var).unwrap(),), None)
# .unwrap();
# })
# }
#
# fn compute(&mut self) {
# println!("Rust calling Python to perform the computation");
# Python::attach(|py| {
# self.model
# .bind(py)
# .call_method("compute", (), None)
# .unwrap();
# })
# }
}
```
Let's break it down in order to perform better error handling:
```rust,no_run
# #![allow(dead_code)]
# use pyo3::prelude::*;
# use pyo3::types::PyList;
#
# pub trait Model {
# fn set_variables(&mut self, inputs: &Vec<f64>);
# fn compute(&mut self);
# fn get_results(&self) -> Vec<f64>;
# }
#
# #[pyclass]
# struct UserModel {
# model: Py<PyAny>,
# }
impl Model for UserModel {
fn get_results(&self) -> Vec<f64> {
println!("Get results from Rust calling Python");
Python::attach(|py| {
let py_result: Bound<'_, PyAny> = self
.model
.bind(py)
.call_method("get_results", (), None)
.unwrap();
if py_result.get_type().name().unwrap() != "list" {
panic!(
"Expected a list for the get_results() method signature, got {}",
py_result.get_type().name().unwrap()
);
}
py_result.extract()
})
.unwrap()
}
# fn set_variables(&mut self, var: &Vec<f64>) {
# println!("Rust calling Python to set the variables");
# Python::attach(|py| {
# let py_model = self.model.bind(py)
# .call_method("set_variables", (PyList::new(py, var).unwrap(),), None)
# .unwrap();
# })
# }
#
# fn compute(&mut self) {
# println!("Rust calling Python to perform the computation");
# Python::attach(|py| {
# self.model
# .bind(py)
# .call_method("compute", (), None)
# .unwrap();
# })
# }
}
```
By doing so, you catch the result of the Python computation and check its type in order to be able to deliver a better error message before performing the unwrapping.
Of course, it does not cover all the possible wrong outputs: the user could return a list of strings instead of a list of floats.
In this case, a runtime panic would still occur due to PyO3, but with an error message much more difficult to decipher for non-rust user.
It is up to the developer exposing the rust code to decide how much effort to invest into Python type error handling and improved error messages.
## The final code
Now let's expose the `solve()` function to make it available from Python.
It is not possible to directly expose the `solve` function to Python, as the type conversion cannot be performed.
It requires an object implementing the `Model` trait as input.
However, the `UserModel` already implements this trait.
Because of this, we can write a function wrapper that takes the `UserModel`--which has already been exposed to Python--as an argument in order to call the core function `solve`.
It is also required to make the struct public.
```rust,no_run
# #![allow(dead_code)]
# fn main() {}
use pyo3::prelude::*;
use pyo3::types::PyList;
pub trait Model {
fn set_variables(&mut self, var: &Vec<f64>);
fn get_results(&self) -> Vec<f64>;
fn compute(&mut self);
}
pub fn solve<T: Model>(model: &mut T) {
println!("Magic solver that mutates the model into a resolved state");
}
#[pyfunction]
#[pyo3(name = "solve")]
pub fn solve_wrapper(model: &mut UserModel) {
solve(model);
}
#[pyclass]
pub struct UserModel {
model: Py<PyAny>,
}
#[pymethods]
impl UserModel {
#[new]
pub fn new(model: Py<PyAny>) -> Self {
UserModel { model }
}
pub fn set_variables(&mut self, var: Vec<f64>) {
println!("Set variables from Python calling Rust");
Model::set_variables(self, &var)
}
pub fn get_results(&mut self) -> Vec<f64> {
println!("Get results from Python calling Rust");
Model::get_results(self)
}
pub fn compute(&mut self) {
Model::compute(self)
}
}
#[pymodule]
mod trait_exposure {
#[pymodule_export]
use super::{UserModel, solve_wrapper};
}
impl Model for UserModel {
fn set_variables(&mut self, var: &Vec<f64>) {
println!("Rust calling Python to set the variables");
Python::attach(|py| {
self.model
.bind(py)
.call_method("set_variables", (PyList::new(py, var).unwrap(),), None)
.unwrap();
})
}
fn get_results(&self) -> Vec<f64> {
println!("Get results from Rust calling Python");
Python::attach(|py| {
let py_result: Bound<'_, PyAny> = self
.model
.bind(py)
.call_method("get_results", (), None)
.unwrap();
if py_result.get_type().name().unwrap() != "list" {
panic!(
"Expected a list for the get_results() method signature, got {}",
py_result.get_type().name().unwrap()
);
}
py_result.extract()
})
.unwrap()
}
fn compute(&mut self) {
println!("Rust calling Python to perform the computation");
Python::attach(|py| {
self.model
.bind(py)
.call_method("compute", (), None)
.unwrap();
})
}
}
```
+89
View File
@@ -0,0 +1,89 @@
# Type stub generation (`*.pyi` files) and introspection
*This feature is still in active development.*
*See [the related issue](https://github.com/PyO3/pyo3/issues/5137).*
*For documentation on type stubs and how to use them with stable PyO3, refer to [this page](python-typing-hints.md)*
PyO3 has a work in progress support to generate [type stub files](https://typing.python.org/en/latest/spec/distributing.html#stub-files).
It works using:
1. PyO3 macros (`#[pyclass]`) that generate constant JSON strings that are then included in the built binaries by rustc if the `experimental-inspect` feature is enabled.
2. The `pyo3-introspection` crate that can parse the generated binaries, extract the JSON strings and build stub files from it.
3. Build tools like `maturin` exposing options in their CLI API to generate the stubs file.
For example, the following Rust code
```rust
#[pymodule]
pub mod example {
use pyo3::prelude::*;
#[pymodule_export]
pub const CONSTANT: &str = "FOO";
#[pyclass(eq)]
#[derive(Eq)]
struct Class {
value: usize
}
#[pymethods]
impl Class {
#[new]
fn new(value: usize) -> Self {
Self { value }
}
#[getter]
fn value(&self) -> usize {
self.value
}
}
#[pyfunction]
#[pyo3(signature = (arg: "list[int]") -> "list[int]")]
fn list_of_int_identity(arg: Bound<'_, PyAny>) -> Bound<'_, PyAny> {
arg
}
}
```
will generate the following stub file:
```python
import typing
CONSTANT: typing.Final = "FOO"
class Class:
def __init__(self, value: int) -> None: ...
@property
def value(self) -> int: ...
def __eq__(self, value: object, /) -> bool: ...
def __ne__(self, value: object, /) -> bool: ...
def list_of_int_identity(arg: "list[int]") -> "list[int]": ...
```
The only piece of new syntax is that the `#[pyo3(signature = ...)]` attribute can contain type annotations like `#[pyo3(signature = (arg: "list[int]") -> "list[int]")]` (note the `""` around type annotations).
This is useful when PyO3 is not able to derive proper type annotations by itself.
To generate stubs file with `maturin` you can use `maturin generate-stubs --output stubs` that will build the project then generate the stubs in the `stubs` directory.
You can also directly integrate the stubs in the built wheels by doing `maturin build --generate-stubs` (works also with `maturin develop`).
PyO3 also provides the smaller `pyo3-introspection` binary that allows to generate stubs from an existing built extension using something like `pyo3-introspection my_module_binary.so my_module_name output` to introspect the `my_module_binary.so` dynamic library for the `my_module_name` Python module and generate its stubs in the `output` directory.
## Constraints and limitations
- The `experimental-inspect` feature is required to generate the introspection fragments.
- Lots of features are not implemented yet.
See [the related issue](https://github.com/PyO3/pyo3/issues/5137) for a list of them.
- Introspection only works with Python modules declared with an inline Rust module.
Modules declared using a function are not supported.
- `FromPyObject::INPUT_TYPE` and `IntoPyObject::OUTPUT_TYPE` must be implemented for PyO3 to get the proper input/output type annotations to use.
- PyO3 is not able to introspect the content of `#[pymodule]` and `#[pymodule_init]` functions.
If they are present, the module is tagged as incomplete using a fake `def __getattr__(name: str) -> Incomplete: ...` function [following best practices](https://typing.python.org/en/latest/guides/writing_stubs.html#incomplete-stubs).
+371
View File
@@ -0,0 +1,371 @@
# Python object types
PyO3 offers two main sets of types to interact with Python objects.
This section of the guide expands into detail about these types and how to choose which to use.
The first set of types are the smart pointers which all Python objects are wrapped in.
These are `Py<T>`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`.
The [first section below](#pyo3s-smart-pointers) expands on each of these in detail and why there are three of them.
The second set of types are types which fill in the generic parameter `T` of the smart pointers.
The most common is `PyAny`, which represents any Python object (similar to Python's `typing.Any`).
There are also concrete types for many Python built-in types, such as `PyList`, `PyDict`, and `PyTuple`.
User defined `#[pyclass]` types also fit this category.
The [second section below](#concrete-python-types) expands on how to use these types.
## PyO3's smart pointers
PyO3's API offers three generic [smart pointers](glossary.md#smart-pointer): `Py<T>`, `Bound<'py, T>` and `Borrowed<'a, 'py, T>`.
For each of these the type parameter `T` will be filled by a [concrete Python type](#concrete-python-types).
For example, a Python list object can be represented by `Py<PyList>`, `Bound<'py, PyList>`, and `Borrowed<'a, 'py, PyList>`.
These smart pointers behave differently due to their lifetime parameters. `Py<T>` has no lifetime parameters, `Bound<'py, T>` has [the `'py` lifetime](./python-from-rust.md#the-py-lifetime) as a parameter, and `Borrowed<'a, 'py, T>` has the `'py` lifetime plus an additional lifetime `'a` to denote the lifetime it is borrowing data for. (You can read more about these lifetimes in the subsections below).
Python objects are reference counted, like [`std::sync::Arc`](https://doc.rust-lang.org/stable/std/sync/struct.Arc.html).
A major reason for these smart pointers is to bring Python's reference counting to a Rust API.
The recommendation of when to use each of these smart pointers is as follows:
- Use `Bound<'py, T>` for as much as possible, as it offers the most efficient and complete API.
- Use `Py<T>` mostly just for storage inside Rust `struct`s which do not want to or can't add a lifetime parameter for `Bound<'py, T>`.
- `Borrowed<'a, 'py, T>` is almost never used.
It is occasionally present at the boundary between Rust and the Python interpreter, for example when borrowing data from Python tuples (which is safe because they are immutable).
The sections below also explain these smart pointers in a little more detail.
### `Py<T>`
[`Py<T>`][Py] is the foundational smart pointer in PyO3's API.
The type parameter `T` denotes the type of the Python object.
Very frequently this is `PyAny`, meaning any Python object.
Because `Py<T>` is not bound to [the `'py` lifetime](./python-from-rust.md#the-py-lifetime), it is the type to use when storing a Python object inside a Rust `struct` or `enum` which do not want to have a lifetime parameter.
In particular, [`#[pyclass]`][pyclass] types are not permitted to have a lifetime, so `Py<T>` is the correct type to store Python objects inside them.
The lack of binding to the `'py` lifetime also carries drawbacks:
- Almost all methods on `Py<T>` require a `Python<'py>` token as the first argument
- Other functionality, such as [`Drop`][Drop], needs to check at runtime for attachment to the Python interpreter, at a small performance cost
Because of the drawbacks `Bound<'py, T>` is preferred for many of PyO3's APIs.
In particular, `Bound<'py, T>` is better for function arguments.
To convert a `Py<T>` into a `Bound<'py, T>`, the `Py::bind` and `Py::into_bound` methods are available. `Bound<'py, T>` can be converted back into `Py<T>` using [`Bound::unbind`].
### `Bound<'py, T>`
[`Bound<'py, T>`][Bound] is the counterpart to `Py<T>` which is also bound to the `'py` lifetime.
It can be thought of as equivalent to the Rust tuple `(Python<'py>, Py<T>)`.
By having the binding to the `'py` lifetime, `Bound<'py, T>` can offer the complete PyO3 API at maximum efficiency.
This means that `Bound<'py, T>` should usually be used whenever carrying this lifetime is acceptable, and `Py<T>` otherwise.
`Bound<'py, T>` engages in Python reference counting.
This means that `Bound<'py, T>` owns a Python object.
Rust code which just wants to borrow a Python object should use a shared reference `&Bound<'py, T>`.
Just like `std::sync::Arc`, using `.clone()` and `drop()` will cheaply increment and decrement the reference count of the object (just in this case, the reference counting is implemented by the Python interpreter itself).
To give an example of how `Bound<'py, T>` is PyO3's primary API type, consider the following Python code:
```python
def example():
x = list() # create a Python list
x.append(1) # append the integer 1 to it
y = x # create a second reference to the list
del x # delete the original reference
```
Using PyO3's API, and in particular `Bound<'py, PyList>`, this code translates into the following Rust code:
```rust
use pyo3::prelude::*;
use pyo3::types::PyList;
fn example<'py>(py: Python<'py>) -> PyResult<()> {
let x: Bound<'py, PyList> = PyList::empty(py);
x.append(1)?;
let y: Bound<'py, PyList> = x.clone(); // y is a new reference to the same list
drop(x); // release the original reference x
Ok(())
}
# Python::attach(example).unwrap();
```
Or, without the type annotations:
```rust
use pyo3::prelude::*;
use pyo3::types::PyList;
fn example(py: Python<'_>) -> PyResult<()> {
let x = PyList::empty(py);
x.append(1)?;
let y = x.clone();
drop(x);
Ok(())
}
# Python::attach(example).unwrap();
```
#### Function argument lifetimes
Because the `'py` lifetime often appears in many function arguments as part of the `Bound<'py, T>` smart pointer, the Rust compiler will often require annotations of input and output lifetimes.
This occurs when the function output has at least one lifetime, and there is more than one lifetime present on the inputs.
To demonstrate, consider this function which takes accepts Python objects and applies the [Python `+` operation][PyAnyMethods::add] to them:
```rust,compile_fail
# use pyo3::prelude::*;
fn add(left: &'_ Bound<'_, PyAny>, right: &'_ Bound<'_, PyAny>) -> PyResult<Bound<'_, PyAny>> {
left.add(right)
}
```
Because the Python `+` operation might raise an exception, this function returns `PyResult<Bound<'_, PyAny>>`.
It doesn't need ownership of the inputs, so it takes `&Bound<'_, PyAny>` shared references.
To demonstrate the point, all lifetimes have used the wildcard `'_` to allow the Rust compiler to attempt to infer them.
Because there are four input lifetimes (two lifetimes of the shared references, and two `'py` lifetimes unnamed inside the `Bound<'_, PyAny>` pointers), the compiler cannot reason about which must be connected to the output.
The correct way to solve this is to add the `'py` lifetime as a parameter for the function, and name all the `'py` lifetimes inside the `Bound<'py, PyAny>` smart pointers.
For the shared references, it's also fine to reduce `&'_` to just `&`.
The working end result is below:
```rust
# use pyo3::prelude::*;
fn add<'py>(
left: &Bound<'py, PyAny>,
right: &Bound<'py, PyAny>,
) -> PyResult<Bound<'py, PyAny>> {
left.add(right)
}
# Python::attach(|py| {
# let s = pyo3::types::PyString::new(py, "s");
# assert!(add(&s, &s).unwrap().eq("ss").unwrap());
# })
```
If naming the `'py` lifetime adds unwanted complexity to the function signature, it is also acceptable to return `Py<PyAny>`, which has no lifetime.
The cost is instead paid by a slight increase in implementation complexity, as seen by the introduction of a call to [`Bound::unbind`]:
```rust
# use pyo3::prelude::*;
fn add(left: &Bound<'_, PyAny>, right: &Bound<'_, PyAny>) -> PyResult<Py<PyAny>> {
let output: Bound<'_, PyAny> = left.add(right)?;
Ok(output.unbind())
}
# Python::attach(|py| {
# let s = pyo3::types::PyString::new(py, "s");
# assert!(add(&s, &s).unwrap().bind(py).eq("ss").unwrap());
# })
```
### `Borrowed<'a, 'py, T>`
[`Borrowed<'a, 'py, T>`][Borrowed] is an advanced type used just occasionally at the edge of interaction with the Python interpreter.
It can be thought of as analogous to the shared reference `&'a Bound<'py, T>`.
The difference is that `Borrowed<'a, 'py, T>` is just a smart pointer rather than a reference-to-a-smart-pointer, which is a helpful reduction in indirection in specific interactions with the Python interpreter.
`Borrowed<'a, 'py, T>` dereferences to `Bound<'py, T>`, so all methods on `Bound<'py, T>` are available on `Borrowed<'a, 'py, T>`.
An example where `Borrowed<'a, 'py, T>` is used is in [`PyTupleMethods::get_borrowed_item`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.PyTupleMethods.html#tymethod.get_item):
```rust
use pyo3::prelude::*;
use pyo3::types::PyTuple;
# fn example<'py>(py: Python<'py>) -> PyResult<()> {
// Create a new tuple with the elements (0, 1, 2)
let t = PyTuple::new(py, [0, 1, 2])?;
for i in 0..=2 {
let entry: Borrowed<'_, 'py, PyAny> = t.get_borrowed_item(i)?;
// `PyAnyMethods::extract` is available on `Borrowed`
// via the dereference to `Bound`
let value: usize = entry.extract()?;
assert_eq!(i, value);
}
# Ok(())
# }
# Python::attach(example).unwrap();
```
### Casting between smart pointer types
To convert between `Py<T>` and `Bound<'py, T>` use the `bind()` / `into_bound()` methods.
Use the `as_unbound()` / `unbind()` methods to go back from `Bound<'py, T>` to `Py<T>`.
```rust,ignore
let obj: Py<PyAny> = ...;
let bound: &Bound<'py, PyAny> = obj.bind(py);
let bound: Bound<'py, PyAny> = obj.into_bound(py);
let obj: &Py<PyAny> = bound.as_unbound();
let obj: Py<PyAny> = bound.unbind();
```
To convert between `Bound<'py, T>` and `Borrowed<'a, 'py, T>` use the `as_borrowed()` method.
`Borrowed<'a, 'py, T>` has a deref coercion to `Bound<'py, T>`.
Use the `to_owned()` method to increment the Python reference count and to create a new `Bound<'py, T>` from the `Borrowed<'a, 'py, T>`.
```rust,ignore
let bound: Bound<'py, PyAny> = ...;
let borrowed: Borrowed<'_, 'py, PyAny> = bound.as_borrowed();
// deref coercion
let bound: &Bound<'py, PyAny> = &borrowed;
// create a new Bound by increase the Python reference count
let bound: Bound<'py, PyAny> = borrowed.to_owned();
```
To convert between `Py<T>` and `Borrowed<'a, 'py, T>` use the `bind_borrowed()` method.
Use either `as_unbound()` or `.to_owned().unbind()` to go back to `Py<T>` from `Borrowed<'a, 'py, T>`, via `Bound<'py, T>`.
```rust,ignore
let obj: Py<PyAny> = ...;
let borrowed: Borrowed<'_, 'py, PyAny> = bound.as_borrowed();
// via deref coercion to Bound and then using Bound::as_unbound
let obj: &Py<PyAny> = borrowed.as_unbound();
// via a new Bound by increasing the Python reference count, and unbind it
let obj: Py<PyAny> = borrowed.to_owned().unbind().
```
## Concrete Python types
In all of `Py<T>`, `Bound<'py, T>`, and `Borrowed<'a, 'py, T>`, the type parameter `T` denotes the type of the Python object referred to by the smart pointer.
This parameter `T` can be filled by:
- [`PyAny`][PyAny], which represents any Python object,
- Native Python types such as `PyList`, `PyTuple`, and `PyDict`, and
- [`#[pyclass]`][pyclass] types defined from Rust
The following subsections covers some further detail about how to work with these types:
- the APIs that are available for these concrete types,
- how to cast `Bound<'py, T>` to a specific concrete type, and
- how to get Rust data out of a `Bound<'py, T>`.
### Using APIs for concrete Python types
Each concrete Python type such as `PyAny`, `PyTuple` and `PyDict` exposes its API on the corresponding bound smart pointer `Bound<'py, PyAny>`, `Bound<'py, PyTuple>` and `Bound<'py, PyDict>`.
Each type's API is exposed as a trait: [`PyAnyMethods`], [`PyTupleMethods`], [`PyDictMethods`], and so on for all concrete types.
Using traits rather than associated methods on the `Bound` smart pointer is done for a couple of reasons:
- Clarity of documentation: each trait gets its own documentation page in the PyO3 API docs.
If all methods were on the `Bound` smart pointer directly, the vast majority of PyO3's API would be on a single, extremely long, documentation page.
- Consistency: downstream code implementing Rust APIs for existing Python types can also follow this pattern of using a trait.
Downstream code would not be allowed to add new associated methods directly on the `Bound` type.
- Future design: it is hoped that a future Rust with [arbitrary self types](https://github.com/rust-lang/rust/issues/44874) will remove the need for these traits in favour of placing the methods directly on `PyAny`, `PyTuple`, `PyDict`, and so on.
These traits are all included in the `pyo3::prelude` module, so with the glob import `use pyo3::prelude::*` the full PyO3 API is made available to downstream code.
The following function accesses the first item in the input Python list, using the `.get_item()` method from the `PyListMethods` trait:
```rust
use pyo3::prelude::*;
use pyo3::types::PyList;
fn get_first_item<'py>(list: &Bound<'py, PyList>) -> PyResult<Bound<'py, PyAny>> {
list.get_item(0)
}
# Python::attach(|py| {
# let l = PyList::new(py, ["hello world"]).unwrap();
# assert!(get_first_item(&l).unwrap().eq("hello world").unwrap());
# })
```
### Casting between Python object types
To cast `Bound<'py, T>` smart pointers to some other type, use the [`.cast()`][Bound::cast] family of functions.
This converts `&Bound<'py, T>` to a different `&Bound<'py, U>`, without transferring ownership.
There is also [`.cast_into()`][Bound::cast_into] to convert `Bound<'py, T>` to `Bound<'py, U>` with transfer of ownership.
These methods are available for all types `T` which implement the [`PyTypeCheck`] trait.
Casting to `Bound<'py, PyAny>` can be done with `.as_any()` or `.into_any()`.
For example, the following snippet shows how to cast `Bound<'py, PyAny>` to `Bound<'py, PyTuple>`:
```rust
# use pyo3::prelude::*;
# use pyo3::types::PyTuple;
# fn example<'py>(py: Python<'py>) -> PyResult<()> {
// create a new Python `tuple`, and use `.into_any()` to erase the type
let obj: Bound<'py, PyAny> = PyTuple::empty(py).into_any();
// use `.cast()` to cast to `PyTuple` without transferring ownership
let _: &Bound<'py, PyTuple> = obj.cast()?;
// use `.cast_into()` to cast to `PyTuple` with transfer of ownership
let _: Bound<'py, PyTuple> = obj.cast_into()?;
# Ok(())
# }
# Python::attach(example).unwrap()
```
Custom [`#[pyclass]`][pyclass] types implement [`PyTypeCheck`], so `.cast()` also works for these types.
The snippet below is the same as the snippet above casting instead to a custom type `MyClass`:
```rust
use pyo3::prelude::*;
#[pyclass]
struct MyClass {}
# fn example<'py>(py: Python<'py>) -> PyResult<()> {
// create a new Python `tuple`, and use `.into_any()` to erase the type
let obj: Bound<'py, PyAny> = Bound::new(py, MyClass {})?.into_any();
// use `.cast()` to cast to `MyClass` without transferring ownership
let _: &Bound<'py, MyClass> = obj.cast()?;
// use `.cast_into()` to cast to `MyClass` with transfer of ownership
let _: Bound<'py, MyClass> = obj.cast_into()?;
# Ok(())
# }
# Python::attach(example).unwrap()
```
### Extracting Rust data from Python objects
To extract Rust data from Python objects, use [`.extract()`][PyAnyMethods::extract] instead of `.cast()`.
This method is available for all types which implement the [`FromPyObject`] trait.
For example, the following snippet extracts a Rust tuple of integers from a Python tuple:
```rust
# use pyo3::prelude::*;
# use pyo3::types::PyTuple;
# fn example<'py>(py: Python<'py>) -> PyResult<()> {
// create a new Python `tuple`, and use `.into_any()` to erase the type
let obj: Bound<'py, PyAny> = PyTuple::new(py, [1, 2, 3])?.into_any();
// extracting the Python `tuple` to a rust `(i32, i32, i32)` tuple
let (x, y, z) = obj.extract::<(i32, i32, i32)>()?;
assert_eq!((x, y, z), (1, 2, 3));
# Ok(())
# }
# Python::attach(example).unwrap()
```
To avoid copying data, [`#[pyclass]`][pyclass] types can directly reference Rust data stored within the Python objects without needing to `.extract()`.
See the [corresponding documentation in the class section of the guide](./class.md#bound-and-interior-mutability) for more detail.
[Bound]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html
[`Bound::unbind`]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html#method.unbind
[Py]: {{#PYO3_DOCS_URL}}/pyo3/struct.Py.html
[PyAnyMethods::add]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.add
[PyAnyMethods::extract]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html#tymethod.extract
[Bound::cast]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html#method.cast
[Bound::cast_into]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html#method.cast_into
[`PyTypeCheck`]: {{#PYO3_DOCS_URL}}/pyo3/type_object/trait.PyTypeCheck.html
[`PyAnyMethods`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyAnyMethods.html
[`PyDictMethods`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyDictMethods.html
[`PyTupleMethods`]: {{#PYO3_DOCS_URL}}/pyo3/types/trait.PyTupleMethods.html
[pyclass]: class.md
[Borrowed]: {{#PYO3_DOCS_URL}}/pyo3/struct.Borrowed.html
[Drop]: https://doc.rust-lang.org/std/ops/trait.Drop.html
[PyAny]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.PyAny.html
[`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html
+25
View File
@@ -0,0 +1,25 @@
.mdbook-tabs {
display: flex;
}
.mdbook-tab {
background-color: var(--table-alternate-bg);
padding: 0.5rem 1rem;
cursor: pointer;
border: none;
font-size: 1.6rem;
line-height: 1.45em;
}
.mdbook-tab.active {
background-color: var(--table-header-bg);
font-weight: bold;
}
.mdbook-tab-content {
padding: 1rem 0rem;
}
.mdbook-tab-content table {
margin: unset;
}
+75
View File
@@ -0,0 +1,75 @@
/**
* Change active tab of tabs.
*
* @param {Element} container
* @param {string} name
*/
const changeTab = (container, name) => {
for (const child of container.children) {
if (!(child instanceof HTMLElement)) {
continue;
}
if (child.classList.contains('mdbook-tabs')) {
for (const tab of child.children) {
if (!(tab instanceof HTMLElement)) {
continue;
}
if (tab.dataset.tabname === name) {
tab.classList.add('active');
} else {
tab.classList.remove('active');
}
}
} else if (child.classList.contains('mdbook-tab-content')) {
if (child.dataset.tabname === name) {
child.classList.remove('hidden');
} else {
child.classList.add('hidden');
}
}
}
};
document.addEventListener('DOMContentLoaded', () => {
const tabs = document.querySelectorAll('.mdbook-tab');
for (const tab of tabs) {
tab.addEventListener('click', () => {
if (!(tab instanceof HTMLElement)) {
return;
}
if (!tab.parentElement || !tab.parentElement.parentElement) {
return;
}
const container = tab.parentElement.parentElement;
const name = tab.dataset.tabname;
const global = container.dataset.tabglobal;
changeTab(container, name);
if (global) {
localStorage.setItem(`mdbook-tabs-${global}`, name);
const globalContainers = document.querySelectorAll(
`.mdbook-tabs-container[data-tabglobal="${global}"]`
);
for (const globalContainer of globalContainers) {
changeTab(globalContainer, name);
}
}
});
}
const containers = document.querySelectorAll('.mdbook-tabs-container[data-tabglobal]');
for (const container of containers) {
const global = container.dataset.tabglobal;
const name = localStorage.getItem(`mdbook-tabs-${global}`);
if (name && document.querySelector(`.mdbook-tab[data-tabname=${name}]`)) {
changeTab(container, name);
}
}
});