Expose a C API for pegen

Grammar/python.gram

@@ -1,6 +1,14 @@
 # Simplified grammar for Python
 
 @bytecode True
+@modulename 'peg_parser' # Non needed for now, but might be needed later
+@trailer '''
+mod_ty
+parse_start(Parser *p)
+{
+    return start_rule(p);
+}
+'''
 
 start[mod_ty]: a=[statements] ENDMARKER { Module(a, NULL, p->arena) }
 statements[asdl_seq*]: a=statement+ { seq_flatten(p, a) }

Include/pegen_interface.h

@@ -0,0 +1,20 @@
+#ifndef Py_LIMITED_API
+#ifndef Py_PEGENINTERFACE
+#define Py_PEGENINTERFACE
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "Python.h"
+#include "Python-ast.h"
+
+PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, PyArena *arena);
+PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(const char *str, PyArena *arena);
+PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, PyArena *arena);
+PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromString(const char *str, PyArena *arena);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_PEGENINTERFACE*/
+#endif /* !Py_LIMITED_API */

Lib/test/test_peg_parser.py

@@ -614,7 +614,7 @@ class ASTGenerationTest(unittest.TestCase):
     def test_correct_ast_generation_on_source_files(self) -> None:
         self.maxDiff = None
         for source in TEST_SOURCES:
-            actual_ast = peg_parser.parse_string(source, mode=1)
+            actual_ast = peg_parser.parse_string(source)
             expected_ast = ast.parse(source)
             self.assertEqual(
                 ast.dump(actual_ast, include_attributes=True),
@@ -625,12 +625,12 @@ def test_correct_ast_generation_on_source_files(self) -> None:
     def test_incorrect_ast_generation_on_source_files(self) -> None:
         for source in FAIL_SOURCES:
             with self.assertRaises(SyntaxError, msg=f"Parsing {source} did not raise an exception"):
-                peg_parser.parse_string(source, mode=0)
+                peg_parser.parse_string(source)
 
     @unittest.expectedFailure
     def test_correct_but_known_to_fail_ast_generation_on_source_files(self) -> None:
         for source in GOOD_BUT_FAIL_SOURCES:
-            actual_ast = peg_parser.parse_string(source, mode=1)
+            actual_ast = peg_parser.parse_string(source)
             expected_ast = ast.parse(source)
             self.assertEqual(
                 ast.dump(actual_ast, include_attributes=True),
@@ -640,7 +640,7 @@ def test_correct_but_known_to_fail_ast_generation_on_source_files(self) -> None:
 
     def test_correct_ast_generation_without_pos_info(self) -> None:
         for source in GOOD_BUT_FAIL_SOURCES:
-            actual_ast = peg_parser.parse_string(source, mode=1)
+            actual_ast = peg_parser.parse_string(source)
             expected_ast = ast.parse(source)
             self.assertEqual(
                 ast.dump(actual_ast),

Makefile.pre.in

@@ -295,6 +295,19 @@ LIBFFI_INCLUDEDIR=	@LIBFFI_INCLUDEDIR@
 
 ##########################################################################
 # Parser
+
+PEGEN_OBJS=		\
+		Parser/pegen/pegen.o \
+		Parser/pegen/parse.o \
+		Parser/pegen/parse_string.o \
+		Parser/pegen/peg_api.o
+
+
+PEGEN_HEADERS= \
+		$(srcdir)/Include/pegen_interface.h \
+		$(srcdir)/Parser/pegen/pegen.h \
+		$(srcdir)/Parser/pegen/parse_string.h
+
 POBJS=		\
 		Parser/acceler.o \
 		Parser/grammar1.o \
@@ -303,9 +316,10 @@ POBJS=		\
 		Parser/parser.o \
 		Parser/token.o \
 
-PARSER_OBJS=	$(POBJS) Parser/myreadline.o Parser/parsetok.o Parser/tokenizer.o
+PARSER_OBJS=	$(POBJS) $(PEGEN_OBJS) Parser/myreadline.o Parser/parsetok.o Parser/tokenizer.o
 
 PARSER_HEADERS= \
+		$(PEGEN_HEADERS) \
 		$(srcdir)/Include/grammar.h \
 		$(srcdir)/Include/parsetok.h \
 		$(srcdir)/Parser/parser.h \
@@ -808,8 +822,8 @@ regen-grammar: regen-token
 .PHONY: regen-pegen
 regen-pegen:
 	PYTHONPATH=$(srcdir)/Tools/peg_generator $(PYTHON_FOR_REGEN) -m pegen -c -q $(srcdir)/Grammar/python.gram \
-		-o $(srcdir)/Modules/peg_parser/parse.new.c
-	$(UPDATE_FILE) $(srcdir)/Modules/peg_parser/parse.c $(srcdir)/Modules/peg_parser/parse.new.c
+		-o $(srcdir)/Parser/pegen/parse.new.c
+	$(UPDATE_FILE) $(srcdir)/Parser/pegen/parse.c $(srcdir)/Parser/pegen/parse.new.c
 
 .PHONY=regen-ast
 regen-ast:

Modules/Setup

@@ -135,7 +135,7 @@ faulthandler faulthandler.c
 _tracemalloc _tracemalloc.c hashtable.c
 
 # PEG-based parser module -- slated to be *the* parser
-peg_parser -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal -I$(srcdir)/Parser -I$(srcdir)/Modules/peg_parser peg_parser/parse.c peg_parser/parse_string.c peg_parser/pegen.c
+peg_parser -DPy_BUILD_CORE_BUILTIN -I$(srcdir)/Include/internal -I$(srcdir)/Parser -I$(srcdir)/Parser/pegen $(srcdir)/Parser/pegen/parse.c peg_parser.c
 
 # The rest of the modules listed in this file are all commented out by
 # default.  Usually they can be detected and built as dynamically

Modules/peg_parser.c

@@ -0,0 +1,73 @@
+#include <Python.h>
+#include <pegen_interface.h>
+
+PyObject *
+_Py_parse_file(PyObject *self, PyObject *args)
+{
+    char *filename;
+
+    if (!PyArg_ParseTuple(args, "s", &filename)) {
+        return NULL;
+    }
+
+    PyArena *arena = PyArena_New();
+    if (arena == NULL) {
+        return NULL;
+    }
+
+    mod_ty res = PyPegen_ASTFromFile(filename, arena);
+    if (res == NULL) {
+        return NULL;
+    }
+    PyObject *result = PyAST_mod2obj(res);
+
+    PyArena_Free(arena);
+    return result;
+}
+
+PyObject *
+_Py_parse_string(PyObject *self, PyObject *args)
+{
+    char *the_string;
+
+    if (!PyArg_ParseTuple(args, "s", &the_string)) {
+        return NULL;
+    }
+
+    PyArena *arena = PyArena_New();
+    if (arena == NULL) {
+        return NULL;
+    }
+
+    mod_ty res = PyPegen_ASTFromString(the_string, arena);
+    if (res == NULL) {
+        return NULL;
+    }
+    PyObject *result = PyAST_mod2obj(res);
+
+    PyArena_Free(arena);
+    return result;
+}
+
+static PyMethodDef ParseMethods[] = {
+    {"parse_file", (PyCFunction)(void(*)(void))_Py_parse_file, METH_VARARGS, "Parse a file."},
+    {"parse_string", (PyCFunction)(void(*)(void))_Py_parse_string, METH_VARARGS, "Parse a string."},
+    {NULL, NULL, 0, NULL}        /* Sentinel */
+};
+
+static struct PyModuleDef parsemodule = {
+    PyModuleDef_HEAD_INIT,
+    .m_name = "peg_parser",
+    .m_doc = "A parser.",
+    .m_methods = ParseMethods,
+};
+
+PyMODINIT_FUNC
+PyInit_peg_parser(void)
+{
+    PyObject *m = PyModule_Create(&parsemodule);
+    if (m == NULL)
+        return NULL;
+
+    return m;
+}

Modules/peg_parser/parse.c → Parser/pegen/parse.c

@@ -1,6 +1,7 @@
 // @generated by pegen.py from ./Grammar/python.gram
 #include "pegen.h"
-static KeywordToken *reserved_keywords[] = {
+const int n_keyword_lists = 9;
+KeywordToken *reserved_keywords[] = {
     NULL,
     NULL,
     (KeywordToken[]) {
@@ -13558,55 +13559,8 @@ _tmp_124_rule(Parser *p)
     return res;
 }
 
-static PyObject *
-parse_file(PyObject *self, PyObject *args, PyObject *kwds)
+mod_ty
+parse_start(Parser *p)
 {
-    static char *keywords[] = {"file", "mode", NULL};
-    const char *filename;
-    int mode = 2;
-
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", keywords, &filename, &mode))
-        return NULL;
-    if (mode < 0 || mode > 2)
-        return PyErr_Format(PyExc_ValueError, "Bad mode, must be 0 <= mode <= 2");
-    return run_parser_from_file(filename, (void *)start_rule, mode, reserved_keywords, 9);
+    return start_rule(p);
 }
-
-static PyObject *
-parse_string(PyObject *self, PyObject *args, PyObject *kwds)
-{
-    static char *keywords[] = {"string", "mode", NULL};
-    const char *the_string;
-    int mode = 2;
-
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", keywords, &the_string, &mode))
-        return NULL;
-    if (mode < 0 || mode > 2)
-        return PyErr_Format(PyExc_ValueError, "Bad mode, must be 0 <= mode <= 2");
-    return run_parser_from_string(the_string, (void *)start_rule, mode, reserved_keywords, 9);
-}
-
-static PyMethodDef ParseMethods[] = {
-    {"parse_file", (PyCFunction)(void(*)(void))parse_file, METH_VARARGS|METH_KEYWORDS, "Parse a file."},
-    {"parse_string", (PyCFunction)(void(*)(void))parse_string, METH_VARARGS|METH_KEYWORDS, "Parse a string."},
-    {NULL, NULL, 0, NULL}        /* Sentinel */
-};
-
-static struct PyModuleDef parsemodule = {
-    PyModuleDef_HEAD_INIT,
-    .m_name = "peg_parser",
-    .m_doc = "A parser.",
-    .m_methods = ParseMethods,
-};
-
-PyMODINIT_FUNC
-PyInit_peg_parser(void)
-{
-    PyObject *m = PyModule_Create(&parsemodule);
-    if (m == NULL)
-        return NULL;
-
-    return m;
-}
-
-// The end

Modules/peg_parser/parse_string.c → Parser/pegen/parse_string.c

@@ -1,6 +1,6 @@
 #include <Python.h>
-#include <tokenizer.h>
 
+#include "../tokenizer.h"
 #include "pegen.h"
 #include "parse_string.h"
 

Parser/pegen/peg_api.c

@@ -0,0 +1,38 @@
+#include <pegen_interface.h>
+
+#include "../tokenizer.h"
+#include "pegen.h"
+
+mod_ty
+PyPegen_ASTFromString(const char *str, PyArena *arena)
+{
+    return run_parser_from_string(str, parse_start, arena);
+}
+
+mod_ty
+PyPegen_ASTFromFile(const char *filename, PyArena *arena)
+{
+    return run_parser_from_file(filename, parse_start, arena);
+}
+
+PyCodeObject *
+PyPegen_CodeObjectFromString(const char *str, PyArena *arena)
+{
+    mod_ty result = run_parser_from_string(str, parse_start, arena);
+    if (result == NULL) {
+        return NULL;
+    }
+
+    return PyAST_CompileObject(result, NULL, NULL, -1, arena);
+}
+
+PyCodeObject *
+PyPegen_CodeObjectFromFile(const char *file, PyArena *arena)
+{
+    mod_ty result = run_parser_from_file(file, parse_start, arena);
+    if (result == NULL) {
+        return NULL;
+    }
+
+    return PyAST_CompileObject(result, NULL, NULL, -1, arena);
+}