Update Rust detection and binding generation

Makefile.pre.in

@@ -1652,10 +1652,6 @@ Makefile Modules/config.c: Makefile.pre \
 	@mv config.c Modules
 	@echo "The Makefile was updated, you may need to re-run make."
 
-.PHONY: regen-rust-wrapper-h
-regen-rust-wrapper-h: $(PYTHON_HEADERS)
-	PYTHON_HEADERS="$(PYTHON_HEADERS)" $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/regen-rust-wrapper-h.py
-
 .PHONY: regen-test-frozenmain
 regen-test-frozenmain: $(BUILDPYTHON)
 	# Regenerate Programs/test_frozenmain.h
@@ -3376,6 +3372,9 @@ Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
 ##########################################################################
 # Module dependencies and platform-specific files
 
+cpython-sys: Modules/cpython-sys/Cargo.toml Modules/cpython-sys/build.rs Modules/cpython-sys/wrapper.h Modules/cpython-sys/parser.h
+	cargo build --lib --locked --package cpython-sys --profile $(CARGO_PROFILE)
+
 # force rebuild when header file or module build flavor (static/shared) is changed
 MODULE_DEPS_STATIC=Modules/config.c
 MODULE_DEPS_SHARED=@MODULE_DEPS_SHARED@

Modules/cpython-sys/build.rs

@@ -4,25 +4,29 @@ use std::path::{Path, PathBuf};
 fn main() {
     let curdir = std::env::current_dir().unwrap();
     let srcdir = curdir.parent().and_then(Path::parent).unwrap();
+    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    generate_c_api_bindings(srcdir, &out_path.as_path());
+    // TODO(emmatyping): generate bindings to the internal parser API
+    // The parser includes things slightly differently, so we should generate
+    // it's bindings independently
+    //generate_parser_bindings(srcdir, &out_path.as_path());
+}
+
+fn generate_c_api_bindings(srcdir: &Path, out_path: &Path) {
     let bindings = bindgen::Builder::default()
         .header("wrapper.h")
         .clang_arg(format!("-I{}", srcdir.as_os_str().to_str().unwrap()))
         .clang_arg(format!("-I{}/Include", srcdir.as_os_str().to_str().unwrap()))
-        .clang_arg(format!("-I{}/Include/internal", srcdir.as_os_str().to_str().unwrap()))
-        .allowlist_function("Py.*")
-        .allowlist_function("_Py.*")
-        .allowlist_type("Py.*")
-        .allowlist_type("_Py.*")
-        .allowlist_var("Py.*")
-        .allowlist_var("_Py.*")
+        .allowlist_function("_?Py.*")
+        .allowlist_type("_?Py.*")
+        .allowlist_var("_?Py.*")
         .blocklist_type("^PyMethodDef$")
         .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))
         .generate()
         .expect("Unable to generate bindings");
 
-    // Write the bindings to the $OUT_DIR/bindings.rs file.
-    let out_path = PathBuf::from(env::var("OUT_DIR").unwrap());
+    // Write the bindings to the $OUT_DIR/c_api.rs file.
     bindings
-        .write_to_file(out_path.join("bindings.rs"))
+        .write_to_file(out_path.join("c_api.rs"))
         .expect("Couldn't write bindings!");
 }

Modules/cpython-sys/parser.h

@@ -0,0 +1,11 @@
+/* Private APIs */
+#define Py_BUILD_CORE
+
+// Parser
+#include "Parser/pegen.h"
+#include "Parser/string_parser.h"
+#include "Parser/lexer/buffer.h"
+#include "Parser/lexer/lexer.h"
+#include "Parser/lexer/state.h"
+#include "Parser/tokenizer/tokenizer.h"
+#include "Parser/tokenizer/helpers.h"

Modules/cpython-sys/src/lib.rs

@@ -6,7 +6,10 @@
 
 use std::ffi::{c_char, c_int, c_void};
 
-include!(concat!(env!("OUT_DIR"), "/bindings.rs"));
+include!(concat!(env!("OUT_DIR"), "/c_api.rs"));
+
+// TODO(emmatyping): include parser bindings (see build.rs)
+//include!(concat!(env!("OUT_DIR"), "/parser.rs"));
 /* Flag passed to newmethodobject */
 /* #define METH_OLDARGS  0x0000   -- unsupported now */
 pub const METH_VARARGS: c_int = 0x0001;

Modules/cpython-sys/wrapper.h

@@ -0,0 +1,177 @@
+/* Public APIs */
+#include "Python.h"
+
+// Misc
+
+// OS macros used in C, not necessary in Rust
+//#include "osdefs.h"
+
+// Valgrind / analysis tools macros and functions
+#include "dynamic_annotations.h"
+// Macros for error codes
+// #include "errcode.h"
+// Macros to define symbol visibility
+// #include "exports.h"
+// Includes pyframe.h and cpython/frameobject.h
+#include "frameobject.h"
+// Includes cpython/marshal.h
+#include "marshal.h"
+// Macros defining opcodes
+#include "opcode.h"
+// More macros defining opcodes
+#include "opcode_ids.h"
+// Dtrace probes
+#include "pydtrace.h"
+//New code should use descrobject.h
+//#include "structmember.h"
+
+// List of all stdlib names, autogenerated
+#include "Python/stdlib_module_names.h"
+
+/* Private APIs */
+#define Py_BUILD_CORE
+
+// Internal
+#include "internal/pycore_parser.h"
+#include "internal/pycore_mimalloc.h"
+#include "internal/mimalloc/mimalloc.h"
+#include "internal/mimalloc/mimalloc/atomic.h"
+#include "internal/mimalloc/mimalloc/internal.h"
+#include "internal/mimalloc/mimalloc/prim.h"
+#include "internal/mimalloc/mimalloc/track.h"
+#include "internal/mimalloc/mimalloc/types.h"
+#include "internal/pycore_abstract.h"
+#include "internal/pycore_asdl.h"
+#include "internal/pycore_ast.h"
+#include "internal/pycore_ast_state.h"
+#include "internal/pycore_atexit.h"
+#include "internal/pycore_audit.h"
+#include "internal/pycore_backoff.h"
+#include "internal/pycore_bitutils.h"
+#include "internal/pycore_blocks_output_buffer.h"
+#include "internal/pycore_brc.h"
+#include "internal/pycore_bytes_methods.h"
+#include "internal/pycore_bytesobject.h"
+#include "internal/pycore_call.h"
+#include "internal/pycore_capsule.h"
+#include "internal/pycore_cell.h"
+#include "internal/pycore_ceval.h"
+#include "internal/pycore_ceval_state.h"
+#include "internal/pycore_code.h"
+#include "internal/pycore_codecs.h"
+#include "internal/pycore_compile.h"
+#include "internal/pycore_complexobject.h"
+#include "internal/pycore_condvar.h"
+#include "internal/pycore_context.h"
+#include "internal/pycore_critical_section.h"
+#include "internal/pycore_crossinterp.h"
+#include "internal/pycore_debug_offsets.h"
+#include "internal/pycore_descrobject.h"
+#include "internal/pycore_dict.h"
+#include "internal/pycore_dict_state.h"
+#include "internal/pycore_dtoa.h"
+#include "internal/pycore_exceptions.h"
+#include "internal/pycore_faulthandler.h"
+#include "internal/pycore_fileutils.h"
+#include "internal/pycore_floatobject.h"
+#include "internal/pycore_flowgraph.h"
+#include "internal/pycore_format.h"
+#include "internal/pycore_frame.h"
+#include "internal/pycore_freelist.h"
+#include "internal/pycore_freelist_state.h"
+#include "internal/pycore_function.h"
+#include "internal/pycore_gc.h"
+#include "internal/pycore_genobject.h"
+#include "internal/pycore_getopt.h"
+#include "internal/pycore_gil.h"
+#include "internal/pycore_global_objects.h"
+#include "internal/pycore_global_objects_fini_generated.h"
+#include "internal/pycore_global_strings.h"
+#include "internal/pycore_hamt.h"
+#include "internal/pycore_hashtable.h"
+#include "internal/pycore_import.h"
+#include "internal/pycore_importdl.h"
+#include "internal/pycore_index_pool.h"
+#include "internal/pycore_initconfig.h"
+#include "internal/pycore_instruments.h"
+#include "internal/pycore_instruction_sequence.h"
+#include "internal/pycore_interp.h"
+#include "internal/pycore_interp_structs.h"
+#include "internal/pycore_interpframe.h"
+#include "internal/pycore_interpframe_structs.h"
+#include "internal/pycore_interpolation.h"
+#include "internal/pycore_intrinsics.h"
+#include "internal/pycore_jit.h"
+#include "internal/pycore_list.h"
+#include "internal/pycore_llist.h"
+#include "internal/pycore_lock.h"
+#include "internal/pycore_long.h"
+#include "internal/pycore_memoryobject.h"
+#include "internal/pycore_mimalloc.h"
+#include "internal/pycore_modsupport.h"
+#include "internal/pycore_moduleobject.h"
+#include "internal/pycore_namespace.h"
+#include "internal/pycore_object.h"
+#include "internal/pycore_object_alloc.h"
+#include "internal/pycore_object_deferred.h"
+#include "internal/pycore_object_stack.h"
+#include "internal/pycore_object_state.h"
+#include "internal/pycore_obmalloc.h"
+#include "internal/pycore_obmalloc_init.h"
+#include "internal/pycore_opcode_metadata.h"
+#include "internal/pycore_opcode_utils.h"
+#include "internal/pycore_optimizer.h"
+#include "internal/pycore_parking_lot.h"
+#include "internal/pycore_parser.h"
+#include "internal/pycore_pathconfig.h"
+#include "internal/pycore_pyarena.h"
+#include "internal/pycore_pyatomic_ft_wrappers.h"
+#include "internal/pycore_pybuffer.h"
+#include "internal/pycore_pyerrors.h"
+#include "internal/pycore_pyhash.h"
+#include "internal/pycore_pylifecycle.h"
+#include "internal/pycore_pymath.h"
+#include "internal/pycore_pymem.h"
+#include "internal/pycore_pymem_init.h"
+#include "internal/pycore_pystate.h"
+#include "internal/pycore_pystats.h"
+#include "internal/pycore_pythonrun.h"
+#include "internal/pycore_pythread.h"
+#include "internal/pycore_qsbr.h"
+#include "internal/pycore_range.h"
+#include "internal/pycore_runtime.h"
+#include "internal/pycore_runtime_init.h"
+#include "internal/pycore_runtime_init_generated.h"
+#include "internal/pycore_runtime_structs.h"
+#include "internal/pycore_semaphore.h"
+#include "internal/pycore_setobject.h"
+#include "internal/pycore_signal.h"
+#include "internal/pycore_sliceobject.h"
+#include "internal/pycore_stats.h"
+#include "internal/pycore_strhex.h"
+#include "internal/pycore_stackref.h"
+#include "internal/pycore_structs.h"
+#include "internal/pycore_structseq.h"
+#include "internal/pycore_symtable.h"
+#include "internal/pycore_sysmodule.h"
+#include "internal/pycore_template.h"
+#include "internal/pycore_time.h"
+#include "internal/pycore_token.h"
+#include "internal/pycore_traceback.h"
+#include "internal/pycore_tracemalloc.h"
+#include "internal/pycore_tstate.h"
+#include "internal/pycore_tuple.h"
+#include "internal/pycore_typedefs.h"
+#include "internal/pycore_typeobject.h"
+#include "internal/pycore_typevarobject.h"
+#include "internal/pycore_ucnhash.h"
+#include "internal/pycore_unicodectype.h"
+#include "internal/pycore_unicodeobject.h"
+#include "internal/pycore_unicodeobject_generated.h"
+#include "internal/pycore_unionobject.h"
+#include "internal/pycore_uniqueid.h"
+#include "internal/pycore_uop.h"
+#include "internal/pycore_uop_ids.h"
+#include "internal/pycore_uop_metadata.h"
+#include "internal/pycore_warnings.h"
+#include "internal/pycore_weakref.h"

Modules/makesetup

@@ -286,7 +286,7 @@ sed -e 's/[ 	]*#.*//' -e '/^[ 	]*$/d' |
 			done
 			libs=
 			# depends on the headers through cpython-sys
-			rule="$objs: \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest Modules/cpython-sys/wrapper.h $prefixed_srcs \$(PYTHON_HEADERS)"
+			rule="$objs: cpython-sys \$(srcdir)/Cargo.toml \$(srcdir)/Cargo.lock \$(srcdir)/$srcdir/$manifest $prefixed_srcs \$(PYTHON_HEADERS)"
 			rule="$rule; cargo build --lib --locked --package ${mods} --profile \$(CARGO_PROFILE)"
 			echo "$rule" >>$rulesf
 			for mod in $mods

Python/remote_debug.h

@@ -29,7 +29,6 @@ extern "C" {
 
 #include "pyconfig.h"
 #include "internal/pycore_ceval.h"
-#include "internal/pycore_debug_offsets.h"
 
 #ifdef __linux__
 #    include <elf.h>