diff --git a/.travis.yml b/.travis.yml index 2d242b4b3c..56682556d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,8 @@ matrix: - PY_CMD=python3 - $PY_CMD -m pip install --user --upgrade pip wheel setuptools install: - - $PY_CMD -m pip install --user --upgrade sphinx sphinx_rtd_theme breathe flake8 pep8-naming pytest + # breathe 4.14 doesn't work with bit fields. See https://github.com/michaeljones/breathe/issues/462 + - $PY_CMD -m pip install --user --upgrade sphinx sphinx_rtd_theme breathe==4.13.1 flake8 pep8-naming pytest - curl -fsSL https://sourceforge.net/projects/doxygen/files/rel-1.8.15/doxygen-1.8.15.linux.bin.tar.gz/download | tar xz - export PATH="$PWD/doxygen-1.8.15/bin:$PATH" script: @@ -137,7 +138,7 @@ matrix: env: PYTHON=2.7 CPP=14 CLANG CMAKE=1 - os: osx name: Python 3.7, c++14, AppleClang 9, Debug build - osx_image: xcode9 + osx_image: xcode9.4 env: PYTHON=3.7 CPP=14 CLANG DEBUG=1 # Test a PyPy 2.7 build - os: linux diff --git a/docs/faq.rst b/docs/faq.rst index 93ccf10e57..4d491fb87f 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -248,6 +248,33 @@ that that were ``malloc()``-ed in another shared library, using data structures with incompatible ABIs, and so on. pybind11 is very careful not to make these types of mistakes. +How can I properly handle Ctrl-C in long-running functions? +=========================================================== + +Ctrl-C is received by the Python interpreter, and holds it until the GIL +is released, so a long-running function won't be interrupted. + +To interrupt from inside your function, you can use the ``PyErr_CheckSignals()`` +function, that will tell if a signal has been raised on the Python side. This +function merely checks a flag, so its impact is negligible. When a signal has +been received, you must either explicitly interrupt execution by throwing +``py::error_already_set`` (which will propagate the existing +``KeyboardInterrupt``), or clear the error (which you usually will not want): + +.. code-block:: cpp + + PYBIND11_MODULE(example, m) + { + m.def("long running_func", []() + { + for (;;) { + if (PyErr_CheckSignals() != 0) + throw py::error_already_set(); + // Long running iteration + } + }); + } + Inconsistent detection of Python version in CMake and pybind11 ============================================================== diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index 6962d6fc53..eda56ae77a 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -116,7 +116,8 @@ enum op_id : int; enum op_type : int; struct undefined_t; template struct op_; -inline void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret); +template +void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret); /// Internal data structure which holds metadata about a keyword argument struct argument_record { @@ -130,11 +131,24 @@ struct argument_record { : name(name), descr(descr), value(value), convert(convert), none(none) { } }; +/// Bundles together arguments for function_record::try_invoke +/// Helps to reduce binary size produced by msvc +struct try_invoke_args +{ + const function_record* ptr; + handle parent; + value_and_holder* self_value_and_holder; + size_t n_args_in; + PyObject* args_in; + PyObject* kwargs_in; + bool convert; +}; + /// Internal data structure which holds metadata about a bound function (signature, overloads, etc.) struct function_record { function_record() : is_constructor(false), is_new_style_constructor(false), is_stateless(false), - is_operator(false), has_args(false), has_kwargs(false), is_method(false) { } + is_operator(false), is_method(false) { } /// Function name char *name = nullptr; /* why no C++ strings? They generate heavier code.. */ @@ -148,8 +162,8 @@ struct function_record { /// List of registered keyword arguments std::vector args; - /// Pointer to lambda function which converts arguments and performs the actual call - handle (*impl) (function_call &) = nullptr; + /// Pointer to lambda function which checks if python arguments satisfy C++ signature, converts them and performs the actual call + handle(*try_invoke)(const try_invoke_args& params) = nullptr; /// Storage for the wrapped function pointer and captured data, if any void *data[3] = { }; @@ -157,6 +171,18 @@ struct function_record { /// Pointer to custom destructor for 'data' (if needed) void (*free_data) (function_record *ptr) = nullptr; + /// Python method object + PyMethodDef *def = nullptr; + + /// Python handle to the parent scope (a class or a module) + handle scope; + + /// Python handle to the sibling function representing an overload chain + handle sibling; + + /// Pointer to next overload + function_record *next = nullptr; + /// Return value policy associated with this function return_value_policy policy = return_value_policy::automatic; @@ -172,29 +198,152 @@ struct function_record { /// True if this is an operator (__add__), etc. bool is_operator : 1; - /// True if the function has a '*args' argument - bool has_args : 1; - - /// True if the function has a '**kwargs' argument - bool has_kwargs : 1; - /// True if this is a method bool is_method : 1; - /// Number of arguments (including py::args and/or py::kwargs, if present) - std::uint16_t nargs; + /// Fill in function_call members, return true we can proceed with execution, false is we should continue + /// with the next candidate + template + PYBIND11_NOINLINE bool prepare_function_call(function_call& call, const try_invoke_args& params) const + { + /* For each overload: + 0. Inject new-style `self` argument + 1. Copy all positional arguments we were given, also checking to make sure that + named positional arguments weren't *also* specified via kwarg. + 2. If we weren't given enough, try to make up the omitted ones by checking + whether they were provided by a kwarg matching the `py::arg("name")` name. If + so, use it (and remove it from kwargs; if not, see if the function binding + provided a default that we can use. + 3. Ensure that either all keyword arguments were "consumed", or that the function + takes a kwargs argument to accept unconsumed kwargs. + 4. Any positional arguments still left get put into a tuple (for args), and any + leftover kwargs get put into a dict. + */ + + size_t pos_args = NumArgs; // Number of positional arguments that we need + if (HasArgs) --pos_args; // (but don't count py::args + if (HasKwargs) --pos_args; // or py::kwargs) + + if (!HasArgs && params.n_args_in > pos_args) + return false; // Too many arguments for this overload + + if (params.n_args_in < pos_args && args.size() < pos_args) + return false; // Not enough arguments given, and not enough defaults to fill in the blanks + + size_t args_to_copy = (std::min)(pos_args, params.n_args_in); // Protect std::min with parentheses + size_t args_copied = 0; + + // 0. Inject new-style `self` argument + if (is_new_style_constructor) { + // The `value` may have been preallocated by an old-style `__init__` + // if it was a preceding candidate for overload resolution. + if (*params.self_value_and_holder) + params.self_value_and_holder->type->dealloc(*params.self_value_and_holder); + + call.init_self = PyTuple_GET_ITEM(params.args_in, 0); + call.args[args_copied] = reinterpret_cast(params.self_value_and_holder); + call.args_convert.set(args_copied, false); + ++args_copied; + } - /// Python method object - PyMethodDef *def = nullptr; + // 1. Copy any position arguments given. + for (; args_copied < args_to_copy; ++args_copied) { + const argument_record* arg_rec = args_copied < args.size() ? &args[args_copied] : nullptr; + if (params.kwargs_in && arg_rec && arg_rec->name && PyDict_GetItemString(params.kwargs_in, arg_rec->name)) { + return false; // Maybe it was meant for another overload (issue #688) + } - /// Python handle to the parent scope (a class or a module) - handle scope; + handle arg(PyTuple_GET_ITEM(params.args_in, args_copied)); - /// Python handle to the sibling function representing an overload chain - handle sibling; + if (arg_rec && !arg_rec->none && arg.is_none()) { + return false; // Maybe it was meant for another overload (issue #688) + } + call.args[args_copied] = arg; + call.args_convert.set(args_copied, params.convert && (arg_rec ? arg_rec->convert : true)); + } - /// Pointer to next overload - function_record *next = nullptr; + // We'll need to copy this if we steal some kwargs for defaults + dict kwargs = reinterpret_borrow(params.kwargs_in); + + // 2. Check kwargs and, failing that, defaults that may help complete the list + if (args_copied < pos_args) { + bool copied_kwargs = false; + + for (; args_copied < pos_args; ++args_copied) { + const auto& arg = args[args_copied]; + + handle value; + if (params.kwargs_in && arg.name) + value = PyDict_GetItemString(kwargs.ptr(), arg.name); + + if (value) { + // Consume a kwargs value + if (!copied_kwargs) { + kwargs = reinterpret_steal(PyDict_Copy(kwargs.ptr())); + copied_kwargs = true; + } + PyDict_DelItemString(kwargs.ptr(), arg.name); + } + else if (arg.value) { + value = arg.value; + } + + if (value) { + call.args[args_copied] = value; + call.args_convert.set(args_copied, params.convert && arg.convert); + } + else + break; + } + + if (args_copied < pos_args) + return false; // Not enough arguments, defaults, or kwargs to fill the positional arguments + } + + // 3. Check everything was consumed (unless we have a kwargs arg) + if (!HasKwargs && kwargs && kwargs.size() > 0) + return false; // Unconsumed kwargs, but no py::kwargs argument to accept them + + // 4a. If we have a py::args argument, create a new tuple with leftovers + if (HasArgs) { + tuple extra_args; + if (args_to_copy == 0) { + // We didn't copy out any position arguments from the args_in tuple, so we + // can reuse it directly without copying: + extra_args = reinterpret_borrow(params.args_in); + } + else if (args_copied >= params.n_args_in) { + extra_args = tuple(0); + } + else { + size_t args_size = params.n_args_in - args_copied; + extra_args = tuple(args_size); + for (size_t i = 0; i < args_size; ++i) { + extra_args[i] = PyTuple_GET_ITEM(params.args_in, args_copied + i); + } + } + call.args[args_copied] = extra_args; + call.args_convert.set(args_copied, false); + call.args_ref = std::move(extra_args); + ++args_copied; + } + + // 4b. If we have a py::kwargs, pass on any remaining kwargs + if (HasKwargs) { + if (!kwargs.ptr()) + kwargs = dict(); // If we didn't get one, send an empty one + call.args[args_copied] = kwargs; + call.args_convert.set(args_copied, false); + call.kwargs_ref = std::move(kwargs); + ++args_copied; + } + +#if !defined(NDEBUG) + if (args_copied != NumArgs) + pybind11_fail("Internal error: function call dispatcher inserted wrong number of arguments!"); +#endif + return true; + } }; /// Special data structure which (temporarily) holds metadata about a bound class @@ -282,12 +431,6 @@ struct type_record { } }; -inline function_call::function_call(const function_record &f, handle p) : - func(f), parent(p) { - args.reserve(f.nargs); - args_convert.reserve(f.nargs); -} - /// Tag for a new-style `__init__` defined in `detail/init.h` struct is_new_style_constructor { }; @@ -303,8 +446,8 @@ template struct process_attribute_default { /// Default implementation: do nothing static void init(const T &, function_record *) { } static void init(const T &, type_record *) { } - static void precall(function_call &) { } - static void postcall(function_call &, handle) { } + template static void precall(function_call &) { } + template static void postcall(function_call &, handle) { } }; /// Process an attribute specifying the function's name @@ -444,14 +587,14 @@ struct process_attribute> : process_attribute_default struct process_attribute> : public process_attribute_default> { - template = 0> - static void precall(function_call &call) { keep_alive_impl(Nurse, Patient, call, handle()); } - template = 0> - static void postcall(function_call &, handle) { } - template = 0> - static void precall(function_call &) { } - template = 0> - static void postcall(function_call &call, handle ret) { keep_alive_impl(Nurse, Patient, call, ret); } + template = 0> + static void precall(function_call &call) { keep_alive_impl(Nurse, Patient, call, handle()); } + template = 0> + static void postcall(function_call &, handle) { } + template = 0> + static void precall(function_call &) { } + template = 0> + static void postcall(function_call &call, handle ret) { keep_alive_impl(Nurse, Patient, call, ret); } }; /// Recursively iterate over variadic template arguments @@ -464,11 +607,13 @@ template struct process_attributes { int unused[] = { 0, (process_attribute::type>::init(args, r), 0) ... }; ignore_unused(unused); } - static void precall(function_call &call) { + template + static void precall(function_call &call) { int unused[] = { 0, (process_attribute::type>::precall(call), 0) ... }; ignore_unused(unused); } - static void postcall(function_call &call, handle fn_ret) { + template + static void postcall(function_call &call, handle fn_ret) { int unused[] = { 0, (process_attribute::type>::postcall(call, fn_ret), 0) ... }; ignore_unused(unused); } diff --git a/include/pybind11/buffer_info.h b/include/pybind11/buffer_info.h index b106d2cc66..1f4115a1fa 100644 --- a/include/pybind11/buffer_info.h +++ b/include/pybind11/buffer_info.h @@ -22,13 +22,14 @@ struct buffer_info { ssize_t ndim = 0; // Number of dimensions std::vector shape; // Shape of the tensor (1 entry per dimension) std::vector strides; // Number of bytes between adjacent entries (for each per dimension) + bool readonly = false; // flag to indicate if the underlying storage may be written to buffer_info() { } buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t ndim, - detail::any_container shape_in, detail::any_container strides_in) + detail::any_container shape_in, detail::any_container strides_in, bool readonly=false) : ptr(ptr), itemsize(itemsize), size(1), format(format), ndim(ndim), - shape(std::move(shape_in)), strides(std::move(strides_in)) { + shape(std::move(shape_in)), strides(std::move(strides_in)), readonly(readonly) { if (ndim != (ssize_t) shape.size() || ndim != (ssize_t) strides.size()) pybind11_fail("buffer_info: ndim doesn't match shape and/or strides length"); for (size_t i = 0; i < (size_t) ndim; ++i) @@ -36,19 +37,23 @@ struct buffer_info { } template - buffer_info(T *ptr, detail::any_container shape_in, detail::any_container strides_in) - : buffer_info(private_ctr_tag(), ptr, sizeof(T), format_descriptor::format(), static_cast(shape_in->size()), std::move(shape_in), std::move(strides_in)) { } + buffer_info(T *ptr, detail::any_container shape_in, detail::any_container strides_in, bool readonly=false) + : buffer_info(private_ctr_tag(), ptr, sizeof(T), format_descriptor::format(), static_cast(shape_in->size()), std::move(shape_in), std::move(strides_in), readonly) { } - buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t size) - : buffer_info(ptr, itemsize, format, 1, {size}, {itemsize}) { } + buffer_info(void *ptr, ssize_t itemsize, const std::string &format, ssize_t size, bool readonly=false) + : buffer_info(ptr, itemsize, format, 1, {size}, {itemsize}, readonly) { } template - buffer_info(T *ptr, ssize_t size) - : buffer_info(ptr, sizeof(T), format_descriptor::format(), size) { } + buffer_info(T *ptr, ssize_t size, bool readonly=false) + : buffer_info(ptr, sizeof(T), format_descriptor::format(), size, readonly) { } + + template + buffer_info(const T *ptr, ssize_t size, bool readonly=true) + : buffer_info(const_cast(ptr), sizeof(T), format_descriptor::format(), size, readonly) { } explicit buffer_info(Py_buffer *view, bool ownview = true) : buffer_info(view->buf, view->itemsize, view->format, view->ndim, - {view->shape, view->shape + view->ndim}, {view->strides, view->strides + view->ndim}) { + {view->shape, view->shape + view->ndim}, {view->strides, view->strides + view->ndim}, view->readonly) { this->view = view; this->ownview = ownview; } @@ -70,6 +75,7 @@ struct buffer_info { strides = std::move(rhs.strides); std::swap(view, rhs.view); std::swap(ownview, rhs.ownview); + readonly = rhs.readonly; return *this; } @@ -81,8 +87,8 @@ struct buffer_info { struct private_ctr_tag { }; buffer_info(private_ctr_tag, void *ptr, ssize_t itemsize, const std::string &format, ssize_t ndim, - detail::any_container &&shape_in, detail::any_container &&strides_in) - : buffer_info(ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in)) { } + detail::any_container &&shape_in, detail::any_container &&strides_in, bool readonly) + : buffer_info(ptr, itemsize, format, ndim, std::move(shape_in), std::move(strides_in), readonly) { } Py_buffer *view = nullptr; bool ownview = false; diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index ad225312d4..0e137a44a6 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -18,6 +18,7 @@ #include #include #include +#include #if defined(PYBIND11_CPP17) # if defined(__has_include) @@ -32,6 +33,10 @@ #include #endif +#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L +# define PYBIND11_HAS_U8STRING +#endif + NAMESPACE_BEGIN(PYBIND11_NAMESPACE) NAMESPACE_BEGIN(detail) @@ -988,6 +993,9 @@ template class type_caster> { template using is_std_char_type = any_of< std::is_same, /* std::string */ +#if defined(PYBIND11_HAS_U8STRING) + std::is_same, /* std::u8string */ +#endif std::is_same, /* std::u16string */ std::is_same, /* std::u32string */ std::is_same /* std::wstring */ @@ -1191,6 +1199,9 @@ template struct string_caster { // Simplify life by being able to assume standard char sizes (the standard only guarantees // minimums, but Python requires exact sizes) static_assert(!std::is_same::value || sizeof(CharT) == 1, "Unsupported char size != 1"); +#if defined(PYBIND11_HAS_U8STRING) + static_assert(!std::is_same::value || sizeof(CharT) == 1, "Unsupported char8_t size != 1"); +#endif static_assert(!std::is_same::value || sizeof(CharT) == 2, "Unsupported char16_t size != 2"); static_assert(!std::is_same::value || sizeof(CharT) == 4, "Unsupported char32_t size != 4"); // wchar_t can be either 16 bits (Windows) or 32 (everywhere else) @@ -1209,7 +1220,7 @@ template struct string_caster { #if PY_MAJOR_VERSION >= 3 return load_bytes(load_src); #else - if (sizeof(CharT) == 1) { + if (std::is_same::value) { return load_bytes(load_src); } @@ -1269,7 +1280,7 @@ template struct string_caster { // without any encoding/decoding attempt). For other C++ char sizes this is a no-op. // which supports loading a unicode from a str, doesn't take this path. template - bool load_bytes(enable_if_t src) { + bool load_bytes(enable_if_t::value, handle> src) { if (PYBIND11_BYTES_CHECK(src.ptr())) { // We were passed a Python 3 raw bytes; accept it into a std::string or char* // without any encoding attempt. @@ -1284,7 +1295,7 @@ template struct string_caster { } template - bool load_bytes(enable_if_t) { return false; } + bool load_bytes(enable_if_t::value, handle>) { return false; } }; template @@ -1885,18 +1896,12 @@ NAMESPACE_BEGIN(detail) struct function_record; /// Internal data associated with a single function call -struct function_call { - function_call(const function_record &f, handle p); // Implementation in attr.h - - /// The function data: - const function_record &func; - - /// Arguments passed to the function: - std::vector args; - - /// The `convert` value the arguments should be loaded with - std::vector args_convert; +template +struct function_call +{ + function_call(handle p) : parent(p) {} +public: /// Extra references for the optional `py::args` and/or `py::kwargs` arguments (which, if /// present, are also in `args` but without a reference). object args_ref, kwargs_ref; @@ -1906,8 +1911,13 @@ struct function_call { /// If this is a call to an initializer, this argument contains `self` handle init_self; -}; + /// Arguments passed to the function: + std::array args; + + /// The `convert` value the arguments should be loaded with + std::bitset args_convert; +}; /// Helper class which loads arguments for C++ functions called from Python template @@ -1927,10 +1937,11 @@ class argument_loader { public: static constexpr bool has_kwargs = kwargs_pos < 0; static constexpr bool has_args = args_pos < 0; + static constexpr size_t num_args = sizeof...(Args); static constexpr auto arg_names = concat(type_descr(make_caster::name)...); - bool load_args(function_call &call) { + bool load_args(function_call& call) { return load_impl_sequence(call, indices{}); } @@ -1947,10 +1958,10 @@ class argument_loader { private: - static bool load_impl_sequence(function_call &, index_sequence<>) { return true; } + static bool load_impl_sequence(function_call&, index_sequence<>) { return true; } template - bool load_impl_sequence(function_call &call, index_sequence) { + bool load_impl_sequence(function_call& call, index_sequence) { for (bool r : {std::get(argcasters).load(call.args[Is], call.args_convert[Is])...}) if (!r) return false; diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 230ae81ae8..edfa7de68c 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -491,6 +491,13 @@ extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int fla view->len = view->itemsize; for (auto s : info->shape) view->len *= s; + view->readonly = info->readonly; + if ((flags & PyBUF_WRITABLE) == PyBUF_WRITABLE && info->readonly) { + if (view) + view->obj = nullptr; + PyErr_SetString(PyExc_BufferError, "Writable buffer requested for readonly storage"); + return -1; + } if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) view->format = const_cast(info->format.c_str()); if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) { diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index bb1affcea3..de10f73b54 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -103,7 +103,7 @@ # endif # pragma warning(push) # pragma warning(disable: 4510 4610 4512 4005) -# if defined(_DEBUG) +# if defined(_DEBUG) && !defined(Py_DEBUG) # define PYBIND11_DEBUG_MARKER # undef _DEBUG # endif @@ -113,6 +113,9 @@ #include #include +/* Python #defines overrides on all sorts of core functions, which + tends to weak havok in C++ codebases that expect these to work + like regular functions (potentially with several overloads) */ #if defined(isalnum) # undef isalnum # undef isalpha @@ -123,6 +126,10 @@ # undef toupper #endif +#if defined(copysign) +# undef copysign +#endif + #if defined(_MSC_VER) # if defined(PYBIND11_DEBUG_MARKER) # define _DEBUG diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 87952daba1..6224dfb226 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -25,6 +25,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass); # define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get((key)) # define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (value)) # define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set((key), nullptr) +# define PYBIND11_TLS_FREE(key) PyThread_tss_free(key) #else // Usually an int but a long on Cygwin64 with Python 3.x # define PYBIND11_TLS_KEY_INIT(var) decltype(PyThread_create_key()) var = 0 @@ -43,6 +44,7 @@ inline PyObject *make_object_base_type(PyTypeObject *metaclass); # define PYBIND11_TLS_REPLACE_VALUE(key, value) \ PyThread_set_key_value((key), (value)) # endif +# define PYBIND11_TLS_FREE(key) (void)key #endif // Python loads modules by default with dlopen with the RTLD_LOCAL flag; under libc++ and possibly @@ -108,6 +110,16 @@ struct internals { #if defined(WITH_THREAD) PYBIND11_TLS_KEY_INIT(tstate); PyInterpreterState *istate = nullptr; + ~internals() { + // This destructor is called *after* Py_Finalize() in finalize_interpreter(). + // That *SHOULD BE* fine. The following details what happens whe PyThread_tss_free is called. + // PYBIND11_TLS_FREE is PyThread_tss_free on python 3.7+. On older python, it does nothing. + // PyThread_tss_free calls PyThread_tss_delete and PyMem_RawFree. + // PyThread_tss_delete just calls TlsFree (on Windows) or pthread_key_delete (on *NIX). Neither + // of those have anything to do with CPython internals. + // PyMem_RawFree *requires* that the `tstate` be allocated with the CPython allocator. + PYBIND11_TLS_FREE(tstate); + } #endif }; @@ -138,7 +150,7 @@ struct type_info { }; /// Tracks the `internals` and `type_info` ABI version independent of the main library version -#define PYBIND11_INTERNALS_VERSION 3 +#define PYBIND11_INTERNALS_VERSION 4 /// On MSVC, debug and release builds are not ABI-compatible! #if defined(_MSC_VER) && defined(_DEBUG) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index d95d61f7bb..71ff6030ce 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -134,7 +134,14 @@ class cpp_function : public function { "The number of argument annotations does not match the number of function arguments"); /* Dispatch code which converts function arguments and performs the actual function call */ - rec->impl = [](function_call &call) -> handle { + rec->try_invoke = [](const try_invoke_args& params) -> handle + { + function_call call(params.parent); + + if (!params.ptr->prepare_function_call(call, params)) + return PYBIND11_TRY_NEXT_OVERLOAD; + + /* Dispatch code which converts function arguments and performs the actual function call */ cast_in args_converter; /* Try to cast the function arguments into the C++ domain */ @@ -145,12 +152,11 @@ class cpp_function : public function { process_attributes::precall(call); /* Get a pointer to the capture object */ - auto data = (sizeof(capture) <= sizeof(call.func.data) - ? &call.func.data : call.func.data[0]); - capture *cap = const_cast(reinterpret_cast(data)); + auto data = (sizeof(capture) <= sizeof(params.ptr->data) ? ¶ms.ptr->data : params.ptr->data[0]); + capture* cap = const_cast(reinterpret_cast(data)); /* Override policy for rvalues -- usually to enforce rvp::move on an rvalue */ - return_value_policy policy = return_value_policy_override::policy(call.func.policy); + return_value_policy policy = return_value_policy_override::policy(params.ptr->policy); /* Function scope guard -- defaults to the compile-to-nothing `void_type` */ using Guard = extract_guard_t; @@ -175,9 +181,6 @@ class cpp_function : public function { /* Register the function with Python from generic (non-templated) code */ initialize_generic(rec, signature.text, types.data(), sizeof...(Args)); - if (cast_in::has_args) rec->has_args = true; - if (cast_in::has_kwargs) rec->has_kwargs = true; - /* Stash some additional information used by an important optimization in 'functional.h' */ using FunctionType = Return (*)(Args...); constexpr bool is_function_ptr = @@ -285,7 +288,6 @@ class cpp_function : public function { #endif rec->signature = strdup(signature.c_str()); rec->args.shrink_to_fit(); - rec->nargs = (std::uint16_t) args; if (rec->sibling && PYBIND11_INSTANCE_METHOD_CHECK(rec->sibling.ptr())) rec->sibling = PYBIND11_INSTANCE_METHOD_GET_FUNCTION(rec->sibling.ptr()); @@ -425,8 +427,7 @@ class cpp_function : public function { using namespace detail; /* Iterator over the list of potentially admissible overloads */ - const function_record *overloads = (function_record *) PyCapsule_GetPointer(self, nullptr), - *it = overloads; + const function_record *overloads = (function_record *) PyCapsule_GetPointer(self, nullptr); /* Need to know how many arguments + keyword arguments there are to pick the right overload */ const size_t n_args_in = (size_t) PyTuple_GET_SIZE(args_in); @@ -451,219 +452,38 @@ class cpp_function : public function { return none().release().ptr(); } + try_invoke_args params = {nullptr, parent, &self_value_and_holder, n_args_in, args_in, kwargs_in, true}; + try { // We do this in two passes: in the first pass, we load arguments with `convert=false`; // in the second, we allow conversion (except for arguments with an explicit // py::arg().noconvert()). This lets us prefer calls without conversion, with // conversion as a fallback. - std::vector second_pass; - - // However, if there are no overloads, we can just skip the no-convert pass entirely - const bool overloaded = it != nullptr && it->next != nullptr; - - for (; it != nullptr; it = it->next) { - - /* For each overload: - 1. Copy all positional arguments we were given, also checking to make sure that - named positional arguments weren't *also* specified via kwarg. - 2. If we weren't given enough, try to make up the omitted ones by checking - whether they were provided by a kwarg matching the `py::arg("name")` name. If - so, use it (and remove it from kwargs; if not, see if the function binding - provided a default that we can use. - 3. Ensure that either all keyword arguments were "consumed", or that the function - takes a kwargs argument to accept unconsumed kwargs. - 4. Any positional arguments still left get put into a tuple (for args), and any - leftover kwargs get put into a dict. - 5. Pack everything into a vector; if we have py::args or py::kwargs, they are an - extra tuple or dict at the end of the positional arguments. - 6. Call the function call dispatcher (function_record::impl) - - If one of these fail, move on to the next overload and keep trying until we get a - result other than PYBIND11_TRY_NEXT_OVERLOAD. - */ - - const function_record &func = *it; - size_t pos_args = func.nargs; // Number of positional arguments that we need - if (func.has_args) --pos_args; // (but don't count py::args - if (func.has_kwargs) --pos_args; // or py::kwargs) - - if (!func.has_args && n_args_in > pos_args) - continue; // Too many arguments for this overload - - if (n_args_in < pos_args && func.args.size() < pos_args) - continue; // Not enough arguments given, and not enough defaults to fill in the blanks - - function_call call(func, parent); - - size_t args_to_copy = (std::min)(pos_args, n_args_in); // Protect std::min with parentheses - size_t args_copied = 0; - - // 0. Inject new-style `self` argument - if (func.is_new_style_constructor) { - // The `value` may have been preallocated by an old-style `__init__` - // if it was a preceding candidate for overload resolution. - if (self_value_and_holder) - self_value_and_holder.type->dealloc(self_value_and_holder); - - call.init_self = PyTuple_GET_ITEM(args_in, 0); - call.args.push_back(reinterpret_cast(&self_value_and_holder)); - call.args_convert.push_back(false); - ++args_copied; - } - - // 1. Copy any position arguments given. - bool bad_arg = false; - for (; args_copied < args_to_copy; ++args_copied) { - const argument_record *arg_rec = args_copied < func.args.size() ? &func.args[args_copied] : nullptr; - if (kwargs_in && arg_rec && arg_rec->name && PyDict_GetItemString(kwargs_in, arg_rec->name)) { - bad_arg = true; - break; - } - handle arg(PyTuple_GET_ITEM(args_in, args_copied)); - if (arg_rec && !arg_rec->none && arg.is_none()) { - bad_arg = true; - break; - } - call.args.push_back(arg); - call.args_convert.push_back(arg_rec ? arg_rec->convert : true); - } - if (bad_arg) - continue; // Maybe it was meant for another overload (issue #688) - - // We'll need to copy this if we steal some kwargs for defaults - dict kwargs = reinterpret_borrow(kwargs_in); - - // 2. Check kwargs and, failing that, defaults that may help complete the list - if (args_copied < pos_args) { - bool copied_kwargs = false; - - for (; args_copied < pos_args; ++args_copied) { - const auto &arg = func.args[args_copied]; - - handle value; - if (kwargs_in && arg.name) - value = PyDict_GetItemString(kwargs.ptr(), arg.name); - - if (value) { - // Consume a kwargs value - if (!copied_kwargs) { - kwargs = reinterpret_steal(PyDict_Copy(kwargs.ptr())); - copied_kwargs = true; - } - PyDict_DelItemString(kwargs.ptr(), arg.name); - } else if (arg.value) { - value = arg.value; - } + // If one of these fail, move on to the next overloadand keep trying until we get a + // result other than PYBIND11_TRY_NEXT_OVERLOAD. - if (value) { - call.args.push_back(value); - call.args_convert.push_back(arg.convert); - } - else - break; - } - - if (args_copied < pos_args) - continue; // Not enough arguments, defaults, or kwargs to fill the positional arguments - } + // However, if there are no overloads, we can just skip the no-convert pass entirely + const bool overloaded = overloads->next != nullptr; - // 3. Check everything was consumed (unless we have a kwargs arg) - if (kwargs && kwargs.size() > 0 && !func.has_kwargs) - continue; // Unconsumed kwargs, but no py::kwargs argument to accept them - - // 4a. If we have a py::args argument, create a new tuple with leftovers - if (func.has_args) { - tuple extra_args; - if (args_to_copy == 0) { - // We didn't copy out any position arguments from the args_in tuple, so we - // can reuse it directly without copying: - extra_args = reinterpret_borrow(args_in); - } else if (args_copied >= n_args_in) { - extra_args = tuple(0); - } else { - size_t args_size = n_args_in - args_copied; - extra_args = tuple(args_size); - for (size_t i = 0; i < args_size; ++i) { - extra_args[i] = PyTuple_GET_ITEM(args_in, args_copied + i); - } + auto try_all_function_records = [&](bool convert) + { + params.convert = convert; + for (params.ptr = overloads; params.ptr != nullptr && result.ptr() == PYBIND11_TRY_NEXT_OVERLOAD; params.ptr = params.ptr->next) { + try { + result = params.ptr->try_invoke(params); } - call.args.push_back(extra_args); - call.args_convert.push_back(false); - call.args_ref = std::move(extra_args); + catch (reference_cast_error&) {} } + }; - // 4b. If we have a py::kwargs, pass on any remaining kwargs - if (func.has_kwargs) { - if (!kwargs.ptr()) - kwargs = dict(); // If we didn't get one, send an empty one - call.args.push_back(kwargs); - call.args_convert.push_back(false); - call.kwargs_ref = std::move(kwargs); - } - - // 5. Put everything in a vector. Not technically step 5, we've been building it - // in `call.args` all along. - #if !defined(NDEBUG) - if (call.args.size() != func.nargs || call.args_convert.size() != func.nargs) - pybind11_fail("Internal error: function call dispatcher inserted wrong number of arguments!"); - #endif - - std::vector second_pass_convert; - if (overloaded) { - // We're in the first no-convert pass, so swap out the conversion flags for a - // set of all-false flags. If the call fails, we'll swap the flags back in for - // the conversion-allowed call below. - second_pass_convert.resize(func.nargs, false); - call.args_convert.swap(second_pass_convert); - } + loader_life_support guard{}; - // 6. Call the function. - try { - loader_life_support guard{}; - result = func.impl(call); - } catch (reference_cast_error &) { - result = PYBIND11_TRY_NEXT_OVERLOAD; - } - - if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) - break; - - if (overloaded) { - // The (overloaded) call failed; if the call has at least one argument that - // permits conversion (i.e. it hasn't been explicitly specified `.noconvert()`) - // then add this call to the list of second pass overloads to try. - for (size_t i = func.is_method ? 1 : 0; i < pos_args; i++) { - if (second_pass_convert[i]) { - // Found one: swap the converting flags back in and store the call for - // the second pass. - call.args_convert.swap(second_pass_convert); - second_pass.push_back(std::move(call)); - break; - } - } - } + if (overloaded) { + try_all_function_records(false); } - if (overloaded && !second_pass.empty() && result.ptr() == PYBIND11_TRY_NEXT_OVERLOAD) { - // The no-conversion pass finished without success, try again with conversion allowed - for (auto &call : second_pass) { - try { - loader_life_support guard{}; - result = call.func.impl(call); - } catch (reference_cast_error &) { - result = PYBIND11_TRY_NEXT_OVERLOAD; - } - - if (result.ptr() != PYBIND11_TRY_NEXT_OVERLOAD) { - // The error reporting logic below expects 'it' to be valid, as it would be - // if we'd encountered this failure in the first-pass loop. - if (!result) - it = &call.func; - break; - } - } - } + try_all_function_records(true); } catch (error_already_set &e) { e.restore(); return nullptr; @@ -771,7 +591,7 @@ class cpp_function : public function { } else if (!result) { std::string msg = "Unable to convert function return value to a " "Python type! The signature was\n\t"; - msg += it->signature; + msg += overloads->signature; append_note_if_missing_header_is_suspected(msg); PyErr_SetString(PyExc_TypeError, msg.c_str()); return nullptr; @@ -1629,13 +1449,14 @@ inline void keep_alive_impl(handle nurse, handle patient) { } } -PYBIND11_NOINLINE inline void keep_alive_impl(size_t Nurse, size_t Patient, function_call &call, handle ret) { +template +PYBIND11_NOINLINE void keep_alive_impl(size_t Nurse, size_t Patient, function_call& call, handle ret) { auto get_arg = [&](size_t n) { if (n == 0) return ret; else if (n == 1 && call.init_self) return call.init_self; - else if (n <= call.args.size()) + else if (n <= NumArgs) return call.args[n - 1]; return handle(); }; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 96eab9662c..4003d69184 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1345,7 +1345,7 @@ class memoryview : public object { buf.strides = py_strides.data(); buf.shape = py_shape.data(); buf.suboffsets = nullptr; - buf.readonly = false; + buf.readonly = info.readonly; buf.internal = nullptr; m_ptr = PyMemoryView_FromBuffer(&buf); diff --git a/pybind11/__init__.py b/pybind11/__init__.py index c625e8c948..4b1de3efaa 100644 --- a/pybind11/__init__.py +++ b/pybind11/__init__.py @@ -2,35 +2,11 @@ def get_include(user=False): - from distutils.dist import Distribution import os - import sys - - # Are we running in a virtual environment? - virtualenv = hasattr(sys, 'real_prefix') or \ - sys.prefix != getattr(sys, "base_prefix", sys.prefix) - - # Are we running in a conda environment? - conda = os.path.exists(os.path.join(sys.prefix, 'conda-meta')) - - if virtualenv: - return os.path.join(sys.prefix, 'include', 'site', - 'python' + sys.version[:3]) - elif conda: - if os.name == 'nt': - return os.path.join(sys.prefix, 'Library', 'include') - else: - return os.path.join(sys.prefix, 'include') + d = os.path.dirname(__file__) + if os.path.exists(os.path.join(d, "include")): + # Package is installed + return os.path.join(d, "include") else: - dist = Distribution({'name': 'pybind11'}) - dist.parse_config_files() - - dist_cobj = dist.get_command_obj('install', create=True) - - # Search for packages in user's home directory? - if user: - dist_cobj.user = user - dist_cobj.prefix = "" - dist_cobj.finalize_options() - - return os.path.dirname(dist_cobj.install_headers) + # Package is from a source directory + return os.path.join(os.path.dirname(d), "include") diff --git a/pybind11/__main__.py b/pybind11/__main__.py index 9ef8378029..89b263a8ad 100644 --- a/pybind11/__main__.py +++ b/pybind11/__main__.py @@ -10,8 +10,7 @@ def print_includes(): dirs = [sysconfig.get_path('include'), sysconfig.get_path('platinclude'), - get_include(), - get_include(True)] + get_include()] # Make unique but preserve order unique_dirs = [] diff --git a/setup.py b/setup.py index f677f2af4a..473ea1ee08 100644 --- a/setup.py +++ b/setup.py @@ -4,40 +4,43 @@ from setuptools import setup from distutils.command.install_headers import install_headers +from distutils.command.build_py import build_py from pybind11 import __version__ import os +package_data = [ + 'include/pybind11/detail/class.h', + 'include/pybind11/detail/common.h', + 'include/pybind11/detail/descr.h', + 'include/pybind11/detail/init.h', + 'include/pybind11/detail/internals.h', + 'include/pybind11/detail/typeid.h', + 'include/pybind11/attr.h', + 'include/pybind11/buffer_info.h', + 'include/pybind11/cast.h', + 'include/pybind11/chrono.h', + 'include/pybind11/common.h', + 'include/pybind11/complex.h', + 'include/pybind11/eigen.h', + 'include/pybind11/embed.h', + 'include/pybind11/eval.h', + 'include/pybind11/functional.h', + 'include/pybind11/iostream.h', + 'include/pybind11/numpy.h', + 'include/pybind11/operators.h', + 'include/pybind11/options.h', + 'include/pybind11/pybind11.h', + 'include/pybind11/pytypes.h', + 'include/pybind11/stl.h', + 'include/pybind11/stl_bind.h', +] + # Prevent installation of pybind11 headers by setting # PYBIND11_USE_CMAKE. if os.environ.get('PYBIND11_USE_CMAKE'): headers = [] else: - headers = [ - 'include/pybind11/detail/class.h', - 'include/pybind11/detail/common.h', - 'include/pybind11/detail/descr.h', - 'include/pybind11/detail/init.h', - 'include/pybind11/detail/internals.h', - 'include/pybind11/detail/typeid.h', - 'include/pybind11/attr.h', - 'include/pybind11/buffer_info.h', - 'include/pybind11/cast.h', - 'include/pybind11/chrono.h', - 'include/pybind11/common.h', - 'include/pybind11/complex.h', - 'include/pybind11/eigen.h', - 'include/pybind11/embed.h', - 'include/pybind11/eval.h', - 'include/pybind11/functional.h', - 'include/pybind11/iostream.h', - 'include/pybind11/numpy.h', - 'include/pybind11/operators.h', - 'include/pybind11/options.h', - 'include/pybind11/pybind11.h', - 'include/pybind11/pytypes.h', - 'include/pybind11/stl.h', - 'include/pybind11/stl_bind.h', - ] + headers = package_data class InstallHeaders(install_headers): @@ -55,6 +58,16 @@ def run(self): self.outfiles.append(out) +# Install the headers inside the package as well +class BuildPy(build_py): + def build_package_data(self): + build_py.build_package_data(self) + for header in package_data: + target = os.path.join(self.build_lib, 'pybind11', header) + self.mkpath(os.path.dirname(target)) + self.copy_file(header, target, preserve_mode=False) + + setup( name='pybind11', version=__version__, @@ -66,7 +79,8 @@ def run(self): packages=['pybind11'], license='BSD', headers=headers, - cmdclass=dict(install_headers=InstallHeaders), + zip_safe=False, + cmdclass=dict(install_headers=InstallHeaders, build_py=BuildPy), classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', diff --git a/tests/constructor_stats.h b/tests/constructor_stats.h index f026e70f98..431e5acef9 100644 --- a/tests/constructor_stats.h +++ b/tests/constructor_stats.h @@ -180,7 +180,7 @@ class ConstructorStats { } } } - catch (const std::out_of_range &) {} + catch (const std::out_of_range&) {} if (!t1) throw std::runtime_error("Unknown class passed to ConstructorStats::get()"); auto &cs1 = get(*t1); // If we have both a t1 and t2 match, one is probably the trampoline class; return whichever diff --git a/tests/test_buffers.cpp b/tests/test_buffers.cpp index 433dfeee61..1bc67ff7b6 100644 --- a/tests/test_buffers.cpp +++ b/tests/test_buffers.cpp @@ -166,4 +166,30 @@ TEST_SUBMODULE(buffers, m) { .def_readwrite("value", (int32_t DerivedBuffer::*) &DerivedBuffer::value) .def_buffer(&DerivedBuffer::get_buffer_info); + struct BufferReadOnly { + const uint8_t value = 0; + BufferReadOnly(uint8_t value): value(value) {} + + py::buffer_info get_buffer_info() { + return py::buffer_info(&value, 1); + } + }; + py::class_(m, "BufferReadOnly", py::buffer_protocol()) + .def(py::init()) + .def_buffer(&BufferReadOnly::get_buffer_info); + + struct BufferReadOnlySelect { + uint8_t value = 0; + bool readonly = false; + + py::buffer_info get_buffer_info() { + return py::buffer_info(&value, 1, readonly); + } + }; + py::class_(m, "BufferReadOnlySelect", py::buffer_protocol()) + .def(py::init<>()) + .def_readwrite("value", &BufferReadOnlySelect::value) + .def_readwrite("readonly", &BufferReadOnlySelect::readonly) + .def_buffer(&BufferReadOnlySelect::get_buffer_info); + } diff --git a/tests/test_buffers.py b/tests/test_buffers.py index f006552bf7..bf7aaed70d 100644 --- a/tests/test_buffers.py +++ b/tests/test_buffers.py @@ -1,8 +1,14 @@ +import io import struct +import sys + import pytest + from pybind11_tests import buffers as m from pybind11_tests import ConstructorStats +PY3 = sys.version_info[0] >= 3 + pytestmark = pytest.requires_numpy with pytest.suppress(ImportError): @@ -85,3 +91,28 @@ def test_pointer_to_member_fn(): buf.value = 0x12345678 value = struct.unpack('i', bytearray(buf))[0] assert value == 0x12345678 + + +@pytest.unsupported_on_pypy +def test_readonly_buffer(): + buf = m.BufferReadOnly(0x64) + view = memoryview(buf) + assert view[0] == 0x64 if PY3 else b'd' + assert view.readonly + + +@pytest.unsupported_on_pypy +def test_selective_readonly_buffer(): + buf = m.BufferReadOnlySelect() + + memoryview(buf)[0] = 0x64 if PY3 else b'd' + assert buf.value == 0x64 + + io.BytesIO(b'A').readinto(buf) + assert buf.value == ord(b'A') + + buf.readonly = True + with pytest.raises(TypeError): + memoryview(buf)[0] = 0 if PY3 else b'\0' + with pytest.raises(TypeError): + io.BytesIO(b'1').readinto(buf) diff --git a/tests/test_builtin_casters.cpp b/tests/test_builtin_casters.cpp index e026127f89..acb2446912 100644 --- a/tests/test_builtin_casters.cpp +++ b/tests/test_builtin_casters.cpp @@ -30,7 +30,7 @@ TEST_SUBMODULE(builtin_casters, m) { else { wstr.push_back((wchar_t) mathbfA32); } // 𝐀, utf32 wstr.push_back(0x7a); // z - m.def("good_utf8_string", []() { return std::string(u8"Say utf8\u203d \U0001f382 \U0001d400"); }); // Say utf8β€½ πŸŽ‚ 𝐀 + m.def("good_utf8_string", []() { return std::string((const char*)u8"Say utf8\u203d \U0001f382 \U0001d400"); }); // Say utf8β€½ πŸŽ‚ 𝐀 m.def("good_utf16_string", [=]() { return std::u16string({ b16, ib16, cake16_1, cake16_2, mathbfA16_1, mathbfA16_2, z16 }); }); // bβ€½πŸŽ‚π€z m.def("good_utf32_string", [=]() { return std::u32string({ a32, mathbfA32, cake32, ib32, z32 }); }); // aπ€πŸŽ‚β€½z m.def("good_wchar_string", [=]() { return wstr; }); // a‽𝐀z @@ -60,6 +60,18 @@ TEST_SUBMODULE(builtin_casters, m) { m.def("strlen", [](char *s) { return strlen(s); }); m.def("string_length", [](std::string s) { return s.length(); }); +#ifdef PYBIND11_HAS_U8STRING + m.attr("has_u8string") = true; + m.def("good_utf8_u8string", []() { return std::u8string(u8"Say utf8\u203d \U0001f382 \U0001d400"); }); // Say utf8β€½ πŸŽ‚ 𝐀 + m.def("bad_utf8_u8string", []() { return std::u8string((const char8_t*)"abc\xd0" "def"); }); + + m.def("u8_char8_Z", []() -> char8_t { return u8'Z'; }); + + // test_single_char_arguments + m.def("ord_char8", [](char8_t c) -> int { return static_cast(c); }); + m.def("ord_char8_lv", [](char8_t &c) -> int { return static_cast(c); }); +#endif + // test_string_view #ifdef PYBIND11_HAS_STRING_VIEW m.attr("has_string_view") = true; @@ -69,9 +81,15 @@ TEST_SUBMODULE(builtin_casters, m) { m.def("string_view_chars", [](std::string_view s) { py::list l; for (auto c : s) l.append((std::uint8_t) c); return l; }); m.def("string_view16_chars", [](std::u16string_view s) { py::list l; for (auto c : s) l.append((int) c); return l; }); m.def("string_view32_chars", [](std::u32string_view s) { py::list l; for (auto c : s) l.append((int) c); return l; }); - m.def("string_view_return", []() { return std::string_view(u8"utf8 secret \U0001f382"); }); + m.def("string_view_return", []() { return std::string_view((const char*)u8"utf8 secret \U0001f382"); }); m.def("string_view16_return", []() { return std::u16string_view(u"utf16 secret \U0001f382"); }); m.def("string_view32_return", []() { return std::u32string_view(U"utf32 secret \U0001f382"); }); + +# ifdef PYBIND11_HAS_U8STRING + m.def("string_view8_print", [](std::u8string_view s) { py::print(s, s.size()); }); + m.def("string_view8_chars", [](std::u8string_view s) { py::list l; for (auto c : s) l.append((std::uint8_t) c); return l; }); + m.def("string_view8_return", []() { return std::u8string_view(u8"utf8 secret \U0001f382"); }); +# endif #endif // test_integer_casting diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index abbfcec493..91422588cf 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -15,6 +15,8 @@ def test_unicode_conversion(): assert m.good_utf16_string() == u"bβ€½πŸŽ‚π€z" assert m.good_utf32_string() == u"aπ€πŸŽ‚β€½z" assert m.good_wchar_string() == u"aβΈ˜π€z" + if hasattr(m, "has_u8string"): + assert m.good_utf8_u8string() == u"Say utf8β€½ πŸŽ‚ 𝐀" with pytest.raises(UnicodeDecodeError): m.bad_utf8_string() @@ -29,12 +31,17 @@ def test_unicode_conversion(): if hasattr(m, "bad_wchar_string"): with pytest.raises(UnicodeDecodeError): m.bad_wchar_string() + if hasattr(m, "has_u8string"): + with pytest.raises(UnicodeDecodeError): + m.bad_utf8_u8string() assert m.u8_Z() == 'Z' assert m.u8_eacute() == u'Γ©' assert m.u16_ibang() == u'β€½' assert m.u32_mathbfA() == u'𝐀' assert m.wchar_heart() == u'β™₯' + if hasattr(m, "has_u8string"): + assert m.u8_char8_Z() == 'Z' def test_single_char_arguments(): @@ -92,6 +99,17 @@ def toobig_message(r): assert m.ord_wchar(u'aa') assert str(excinfo.value) == toolong_message + if hasattr(m, "has_u8string"): + assert m.ord_char8(u'a') == 0x61 # simple ASCII + assert m.ord_char8_lv(u'b') == 0x62 + assert m.ord_char8(u'Γ©') == 0xE9 # requires 2 bytes in utf-8, but can be stuffed in a char + with pytest.raises(ValueError) as excinfo: + assert m.ord_char8(u'Δ€') == 0x100 # requires 2 bytes, doesn't fit in a char + assert str(excinfo.value) == toobig_message(0x100) + with pytest.raises(ValueError) as excinfo: + assert m.ord_char8(u'ab') + assert str(excinfo.value) == toolong_message + def test_bytes_to_string(): """Tests the ability to pass bytes to C++ string-accepting functions. Note that this is @@ -116,10 +134,15 @@ def test_string_view(capture): assert m.string_view_chars("Hi πŸŽ‚") == [72, 105, 32, 0xf0, 0x9f, 0x8e, 0x82] assert m.string_view16_chars("Hi πŸŽ‚") == [72, 105, 32, 0xd83c, 0xdf82] assert m.string_view32_chars("Hi πŸŽ‚") == [72, 105, 32, 127874] + if hasattr(m, "has_u8string"): + assert m.string_view8_chars("Hi") == [72, 105] + assert m.string_view8_chars("Hi πŸŽ‚") == [72, 105, 32, 0xf0, 0x9f, 0x8e, 0x82] assert m.string_view_return() == "utf8 secret πŸŽ‚" assert m.string_view16_return() == "utf16 secret πŸŽ‚" assert m.string_view32_return() == "utf32 secret πŸŽ‚" + if hasattr(m, "has_u8string"): + assert m.string_view8_return() == "utf8 secret πŸŽ‚" with capture: m.string_view_print("Hi") @@ -132,6 +155,14 @@ def test_string_view(capture): utf16 πŸŽ‚ 8 utf32 πŸŽ‚ 7 """ + if hasattr(m, "has_u8string"): + with capture: + m.string_view8_print("Hi") + m.string_view8_print("utf8 πŸŽ‚") + assert capture == """ + Hi 2 + utf8 πŸŽ‚ 9 + """ with capture: m.string_view_print("Hi, ascii") @@ -144,6 +175,14 @@ def test_string_view(capture): Hi, utf16 πŸŽ‚ 12 Hi, utf32 πŸŽ‚ 11 """ + if hasattr(m, "has_u8string"): + with capture: + m.string_view8_print("Hi, ascii") + m.string_view8_print("Hi, utf8 πŸŽ‚") + assert capture == """ + Hi, ascii 9 + Hi, utf8 πŸŽ‚ 13 + """ def test_integer_casting():