pybind11

pybind11 Overview

pybind11 is a lightweight header-only library that exposes C++ types in Python and vice versa, primarily used to create Python extension modules from existing C++ code. It leverages C++11 features (variadic templates, move semantics) to provide a clean binding syntax without the boilerplate of the raw Python C API or older tools like Boost.Python.

From a CMake perspective, pybind11 provides the pybind11_add_module() function — a specialized wrapper around add_library(MODULE ...) that automatically handles Python include paths, library linking, extension suffixes, and platform-specific flags.

FetchContent vs find_package

pybind11 supports both acquisition methods. FetchContent is preferred for reproducibility, while find_package works when pybind11 is installed via pip:

# Option A: FetchContent (recommended for reproducibility)
cmake_minimum_required(VERSION 3.20)
project(mymodule LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)

include(FetchContent)
FetchContent_Declare(
    pybind11
    GIT_REPOSITORY https://github.com/pybind/pybind11.git
    GIT_TAG        v2.12.0
    GIT_SHALLOW    TRUE
)
FetchContent_MakeAvailable(pybind11)

pybind11_add_module(mymodule src/bindings.cpp)
target_link_libraries(mymodule PRIVATE mylib)

# Option B: find_package (after pip install pybind11)
cmake_minimum_required(VERSION 3.20)
project(mymodule LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)

# pybind11 installed via: pip install pybind11
find_package(pybind11 2.12 REQUIRED)

pybind11_add_module(mymodule src/bindings.cpp)
target_link_libraries(mymodule PRIVATE mylib)

# Install pybind11 for find_package discovery
pip install pybind11

# Configure with the correct Python
cmake -B build -S . \
    -Dpybind11_DIR=$(python -c "import pybind11; print(pybind11.get_cmake_dir())")

# Or set Python explicitly
cmake -B build -S . \
    -DPython_EXECUTABLE=$(which python3) \
    -DPYTHON_EXECUTABLE=$(which python3)

pybind11_add_module()

The pybind11_add_module() function is the primary CMake integration point. It creates a MODULE library target with all necessary Python configuration:

# Basic module definition
pybind11_add_module(mymodule
    src/bindings.cpp
    src/math_bindings.cpp
    src/io_bindings.cpp
)

# Link against your C++ library
target_link_libraries(mymodule PRIVATE
    mylib          # Your C++ library
    Eigen3::Eigen  # Other dependencies
)

# Set output directory for development (importable from build dir)
set_target_properties(mymodule PROPERTIES
    LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/mypackage
)

# Install the module
install(TARGETS mymodule
    LIBRARY DESTINATION ${Python_SITEARCH}/mypackage
)

# Advanced: multiple modules in one project
pybind11_add_module(_core src/core_bindings.cpp)
pybind11_add_module(_math src/math_bindings.cpp)
pybind11_add_module(_io src/io_bindings.cpp)

foreach(mod _core _math _io)
    target_link_libraries(${mod} PRIVATE mylib)
    set_target_properties(${mod} PROPERTIES
        LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/mypackage
    )
endforeach()

Binding Classes and Functions

The C++ binding code uses pybind11 macros and helper classes to expose your API to Python:

// src/bindings.cpp — Complete binding example
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>          // std::vector, std::map auto-conversion
#include <pybind11/operators.h>    // operator overloading
#include "matrix.h"
#include "solver.h"

namespace py = pybind11;

// Bind a simple function
double compute_norm(const std::vector<double>& vec) {
    double sum = 0.0;
    for (double v : vec) sum += v * v;
    return std::sqrt(sum);
}

PYBIND11_MODULE(mymodule, m) {
    m.doc() = "My high-performance math module";

    // Free functions
    m.def("compute_norm", &compute_norm,
          py::arg("vec"),
          "Compute the L2 norm of a vector");

    // Class binding
    py::class_<Matrix>(m, "Matrix")
        .def(py::init<int, int>(),
             py::arg("rows"), py::arg("cols"))
        .def("rows", &Matrix::rows)
        .def("cols", &Matrix::cols)
        .def("get", &Matrix::get,
             py::arg("i"), py::arg("j"))
        .def("set", &Matrix::set,
             py::arg("i"), py::arg("j"), py::arg("value"))
        .def("transpose", &Matrix::transpose)
        .def("__repr__", [](const Matrix& m) {
            return "<Matrix " + std::to_string(m.rows()) +
                   "x" + std::to_string(m.cols()) + ">";
        })
        // Operator overloading
        .def(py::self + py::self)
        .def(py::self * float());

    // Enum binding
    py::enum_<Solver::Method>(m, "SolverMethod")
        .value("LU", Solver::Method::LU)
        .value("QR", Solver::Method::QR)
        .value("SVD", Solver::Method::SVD)
        .export_values();

    // Class with enum parameter
    py::class_<Solver>(m, "Solver")
        .def(py::init<Solver::Method>(),
             py::arg("method") = Solver::Method::LU)
        .def("solve", &Solver::solve,
             py::arg("A"), py::arg("b"));
}

NumPy Integration

pybind11's NumPy support enables zero-copy data exchange between C++ and Python arrays, critical for scientific computing performance:

// src/numpy_bindings.cpp — NumPy array integration
#include <pybind11/pybind11.h>
#include <pybind11/numpy.h>
#include <cmath>

namespace py = pybind11;

// Accept and return NumPy arrays (zero-copy when possible)
py::array_t<double> multiply_elementwise(
    py::array_t<double> input1,
    py::array_t<double> input2)
{
    auto buf1 = input1.request();
    auto buf2 = input2.request();

    if (buf1.size != buf2.size)
        throw std::runtime_error("Input shapes must match");

    auto result = py::array_t<double>(buf1.size);
    auto buf_result = result.request();

    double* ptr1 = static_cast<double*>(buf1.ptr);
    double* ptr2 = static_cast<double*>(buf2.ptr);
    double* ptr_result = static_cast<double*>(buf_result.ptr);

    for (ssize_t i = 0; i < buf1.size; i++) {
        ptr_result[i] = ptr1[i] * ptr2[i];
    }

    return result;
}

// Unchecked access for maximum performance (no bounds checking)
py::array_t<double> fast_normalize(py::array_t<double, py::array::c_style> input) {
    auto r = input.unchecked<1>();  // 1D array, no bounds checks
    auto result = py::array_t<double>(r.size());
    auto w = result.mutable_unchecked<1>();

    double norm = 0.0;
    for (ssize_t i = 0; i < r.size(); i++)
        norm += r(i) * r(i);
    norm = std::sqrt(norm);

    for (ssize_t i = 0; i < r.size(); i++)
        w(i) = r(i) / norm;

    return result;
}

PYBIND11_MODULE(_numpy_ops, m) {
    m.def("multiply_elementwise", &multiply_elementwise,
          py::arg("a"), py::arg("b"),
          "Element-wise multiplication of two arrays");
    m.def("fast_normalize", &fast_normalize,
          py::arg("input"),
          "Normalize a vector (L2 norm)");
}

# CMakeLists.txt — NumPy-aware module
find_package(Python 3.9 REQUIRED COMPONENTS Interpreter Development NumPy)

pybind11_add_module(_numpy_ops src/numpy_bindings.cpp)
target_link_libraries(_numpy_ops PRIVATE Python::NumPy)

scikit-build-core for Distribution

scikit-build-core is the modern standard for packaging CMake-based Python extensions. It replaces the legacy setup.py approach with a pyproject.toml-driven workflow:

# pyproject.toml — scikit-build-core configuration
[build-system]
requires = ["scikit-build-core>=0.8", "pybind11>=2.12"]
build-backend = "scikit_build_core.build"

[project]
name = "mypackage"
version = "1.0.0"
requires-python = ">=3.9"
dependencies = ["numpy>=1.21"]

[tool.scikit-build]
cmake.minimum-version = "3.20"
cmake.build-type = "Release"
wheel.packages = ["src/mypackage"]

[tool.scikit-build.cmake.define]
CMAKE_CXX_STANDARD = "17"

# CMakeLists.txt — scikit-build-core compatible
cmake_minimum_required(VERSION 3.20)
project(mypackage LANGUAGES CXX)

set(CMAKE_CXX_STANDARD 17)

find_package(pybind11 CONFIG REQUIRED)
find_package(Python REQUIRED COMPONENTS Interpreter Development.Module NumPy)

pybind11_add_module(_core src/bindings.cpp)
target_link_libraries(_core PRIVATE mylib)

install(TARGETS _core DESTINATION mypackage)

# Build and install the package
pip install -e .          # Editable install for development
pip install .             # Standard install
pip wheel .               # Build wheel for distribution
python -m build           # Build sdist + wheel

Stub Generation

Type stubs (.pyi files) provide IDE autocompletion and type checking for compiled extensions. pybind11-stubgen generates these automatically:

# Install stub generator
pip install pybind11-stubgen

# Generate stubs after building your module
pybind11-stubgen mypackage._core -o src/

# Or integrate into CMake as a post-build step

# CMakeLists.txt — Automatic stub generation
find_package(Python REQUIRED COMPONENTS Interpreter)

pybind11_add_module(_core src/bindings.cpp)

# Generate stubs after building
add_custom_command(TARGET _core POST_BUILD
    COMMAND ${Python_EXECUTABLE} -m pybind11_stubgen
            mypackage._core
            --output-dir ${CMAKE_SOURCE_DIR}/src
    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
    COMMENT "Generating type stubs for _core"
    VERBATIM
)

# Generated stub: src/mypackage/_core.pyi
from typing import List, overload
import numpy as np
import numpy.typing as npt

def compute_norm(vec: List[float]) -> float: ...
def fast_normalize(input: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: ...

class Matrix:
    def __init__(self, rows: int, cols: int) -> None: ...
    def rows(self) -> int: ...
    def cols(self) -> int: ...
    def get(self, i: int, j: int) -> float: ...
    def set(self, i: int, j: int, value: float) -> None: ...
    def transpose(self) -> "Matrix": ...
    @overload
    def __add__(self, other: "Matrix") -> "Matrix": ...
    @overload
    def __mul__(self, scalar: float) -> "Matrix": ...

Debugging Extensions

Debugging compiled Python extensions requires building with debug symbols and attaching a native debugger to the Python process:

# Debug build configuration
cmake_minimum_required(VERSION 3.20)
project(mymodule LANGUAGES CXX)

# Force debug info even in Release builds
if(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" OR CMAKE_BUILD_TYPE STREQUAL "Debug")
    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
endif()

find_package(pybind11 REQUIRED)

pybind11_add_module(_core src/bindings.cpp)

# For Debug builds, disable optimizations for easier stepping
target_compile_options(_core PRIVATE
    $<$<CONFIG:Debug>:-O0 -g -UNDEBUG>
)

# Build with debug symbols
cmake -B build -S . -DCMAKE_BUILD_TYPE=Debug
cmake --build build

# Method 1: Attach GDB to Python
gdb -ex run --args python -c "import mypackage; mypackage.compute_norm([1,2,3])"

# Method 2: Start Python under GDB
gdb python
(gdb) run -c "import mypackage; mypackage.compute_norm([1,2,3])"
(gdb) break bindings.cpp:42
(gdb) continue

# Method 3: VS Code launch.json for mixed Python/C++ debugging
# Use "Python C++ Debugger" extension