Rewrite run_parser* functions to always return a mod_ty

Grammar/python.gram

@@ -8,6 +8,8 @@ parse_start(Parser *p)
 {
     return start_rule(p);
 }
+
+// The end
 '''
 
 start[mod_ty]: a=[statements] ENDMARKER { Module(a, NULL, p->arena) }

Include/pegen_interface.h

@@ -8,10 +8,10 @@ extern "C" {
 #include "Python.h"
 #include "Python-ast.h"
 
-PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, PyArena *arena);
-PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(const char *str, PyArena *arena);
-PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, PyArena *arena);
-PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromString(const char *str, PyArena *arena);
+PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, PyObject *filename_ob, PyArena *arena);
+PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(const char *str, PyObject *filename_ob, PyArena *arena);
+PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, PyObject *filename_ob, PyArena *arena);
+PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromString(const char *str, PyObject *filename_ob, PyArena *arena);
 
 #ifdef __cplusplus
 }

Modules/peg_parser.c

@@ -15,13 +15,21 @@ _Py_parse_file(PyObject *self, PyObject *args)
         return NULL;
     }
 
-    mod_ty res = PyPegen_ASTFromFile(filename, arena);
+    PyObject *result = NULL;
+
+    PyObject *filename_ob = PyUnicode_FromString(filename);
+    if (filename_ob == NULL) {
+        goto error;
+    }
+
+    mod_ty res = PyPegen_ASTFromFile(filename, filename_ob, arena);
     if (res == NULL) {
-        PyArena_Free(arena);
-        return NULL;
+        goto error;
     }
-    PyObject *result = PyAST_mod2obj(res);
+    result = PyAST_mod2obj(res);
 
+error:
+    Py_XDECREF(filename_ob);
     PyArena_Free(arena);
     return result;
 }
@@ -40,13 +48,21 @@ _Py_parse_string(PyObject *self, PyObject *args)
         return NULL;
     }
 
-    mod_ty res = PyPegen_ASTFromString(the_string, arena);
+    PyObject *result = NULL;
+
+    PyObject *filename_ob = PyUnicode_FromString("<string>");
+    if (filename_ob == NULL) {
+        goto error;
+    }
+
+    mod_ty res = PyPegen_ASTFromString(the_string, filename_ob, arena);
     if (res == NULL) {
-        PyArena_Free(arena);
-        return NULL;
+        goto error;
     }
-    PyObject *result = PyAST_mod2obj(res);
+    result = PyAST_mod2obj(res);
 
+error:
+    Py_XDECREF(filename_ob);
     PyArena_Free(arena);
     return result;
 }

Parser/pegen/parse.c

@@ -13564,3 +13564,5 @@ parse_start(Parser *p)
 {
     return start_rule(p);
 }
+
+// The end

Parser/pegen/peg_api.c

@@ -4,25 +4,35 @@
 #include "pegen.h"
 
 mod_ty
-PyPegen_ASTFromString(const char *str, PyArena *arena)
+PyPegen_ASTFromString(const char *str, PyObject *filename_ob, PyArena *arena)
 {
-    return run_parser_from_string(str, parse_start, RAW_AST_OBJECT, arena);
+    return run_parser_from_string(str, parse_start, filename_ob, arena);
 }
 
 mod_ty
-PyPegen_ASTFromFile(const char *filename, PyArena *arena)
+PyPegen_ASTFromFile(const char *filename, PyObject *filename_ob, PyArena *arena)
 {
-    return run_parser_from_file(filename, parse_start, RAW_AST_OBJECT, arena);
+    return run_parser_from_file(filename, parse_start, filename_ob, arena);
 }
 
 PyCodeObject *
-PyPegen_CodeObjectFromString(const char *str, PyArena *arena)
+PyPegen_CodeObjectFromString(const char *str, PyObject *filename_ob, PyArena *arena)
 {
-    return run_parser_from_string(str, parse_start, CODE_OBJECT, arena);
+    mod_ty res = run_parser_from_string(str, parse_start, filename_ob, arena);
+    if (res == NULL) {
+        return NULL;
+    }
+
+    return PyAST_CompileObject(res, filename_ob, NULL, -1, arena);
 }
 
 PyCodeObject *
-PyPegen_CodeObjectFromFile(const char *file, PyArena *arena)
+PyPegen_CodeObjectFromFile(const char *filename, PyObject *filename_ob, PyArena *arena)
 {
-    return run_parser_from_file(file, parse_start, CODE_OBJECT, arena);
+    mod_ty res = run_parser_from_string(filename, parse_start, filename_ob, arena);
+    if (res == NULL) {
+        return NULL;
+    }
+
+    return PyAST_CompileObject(res, filename_ob, NULL, -1, arena);
 }

Parser/pegen/pegen.c

@@ -536,8 +536,8 @@ Parser_New(struct tok_state *tok, mod_ty (*parse_func)(Parser *), int input_mode
     return p;
 }
 
-void *
-run_parser(Parser *p, int mode)
+mod_ty
+run_parser(Parser *p)
 {
     int error = setjmp(p->error_env);
     if (error) {
@@ -559,89 +559,65 @@ run_parser(Parser *p, int mode)
         return NULL;
     }
 
-    void *result = NULL;
-    if (mode == CODE_OBJECT) {
-        result = PyAST_CompileObject(res, p->tok->filename, NULL, -1, p->arena);
-    }
-    else if (mode == AST_OBJECT) {
-        result = PyAST_mod2obj(res);
-    }
-    else {
-        result = res;
-    }
-
-    return result;
+    return res;
 }
 
-void *
-run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), int mode,
-                     PyArena *arena)
+mod_ty
+run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena)
 {
     FILE *fp = fopen(filename, "rb");
     if (fp == NULL) {
         PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename);
         return NULL;
     }
 
-    PyObject *filename_ob = NULL;
-    if ((filename_ob = PyUnicode_FromString(filename)) == NULL) {
-        return NULL;
-    }
-
     // From here on we need to clean up even if there's an error
-    void *result = NULL;
+    mod_ty result = NULL;
 
     struct tok_state *tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
     if (tok == NULL) {
         goto error;
     }
-
-    // Transfers ownership
     tok->filename = filename_ob;
-    filename_ob = NULL;
+    Py_INCREF(filename_ob);
 
     Parser *p = Parser_New(tok, parse_func, FILE_INPUT, arena);
     if (p == NULL) {
-        PyTokenizer_Free(tok);
-        goto error;
+        goto after_tok_error;
     }
 
-    result = run_parser(p, mode);
-
+    result = run_parser(p);
     Parser_Free(p);
-    PyTokenizer_Free(tok);
 
+after_tok_error:
+    PyTokenizer_Free(tok);
 error:
     fclose(fp);
-    Py_XDECREF(filename_ob);
     return result;
 }
 
-void *
-run_parser_from_string(const char *str, mod_ty (*parse_func)(Parser *), int mode,
-                       PyArena *arena)
+mod_ty
+run_parser_from_string(const char *str, mod_ty(*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena)
 {
-    void *result = NULL;
     struct tok_state *tok = PyTokenizer_FromString(str, 1);
     if (tok == NULL) {
         return NULL;
     }
+    tok->filename = filename_ob;
+    Py_INCREF(filename_ob);
 
-    tok->filename = PyUnicode_FromString("<string>");
-    if (tok->filename == NULL) {
-        PyTokenizer_Free(tok);
-        return NULL;
-    }
+    // We need to clear up from here on
+    mod_ty result = NULL;
 
     Parser *p = Parser_New(tok, parse_func, STRING_INPUT, arena);
     if (p == NULL) {
-        PyTokenizer_Free(tok);
-        return NULL;
+        goto error;
     }
 
-    result = run_parser(p, mode);
-
+    result = run_parser(p);
     Parser_Free(p);
+
+error:
     PyTokenizer_Free(tok);
     return result;
 }

Parser/pegen/pegen.h

@@ -14,13 +14,6 @@ enum INPUT_MODE {
 };
 typedef enum INPUT_MODE INPUT_MODE;
 
-enum MODE {
-    RAW_AST_OBJECT,
-    AST_OBJECT,
-    CODE_OBJECT
-};
-typedef enum MODE MODE;
-
 typedef struct _memo {
     int type;
     void *node;
@@ -143,8 +136,8 @@ CHECK_CALL_NULL_ALLOWED(Parser *p, void *result)
 #define CHECK_NULL_ALLOWED(result) CHECK_CALL_NULL_ALLOWED(p, result)
 
 PyObject *new_identifier(Parser *, char *);
-void *run_parser_from_file(const char *, mod_ty(*)(Parser *), int, PyArena *);
-void *run_parser_from_string(const char *, mod_ty(*)(Parser *), int, PyArena *);
+mod_ty run_parser_from_file(const char *, mod_ty(*)(Parser *), PyObject *, PyArena *);
+mod_ty run_parser_from_string(const char *, mod_ty(*)(Parser *), PyObject *, PyArena *);
 asdl_seq *singleton_seq(Parser *, void *);
 asdl_seq *seq_insert_in_front(Parser *, void *, asdl_seq *);
 asdl_seq *seq_flatten(Parser *, asdl_seq *);

Python/pythonrun.c

@@ -1036,7 +1036,7 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals,
     // I left this here so when you run 'nm python | grep PyPegen' you can see
     // that the symbol is included and this function is called. Obviously this is
     // wrong, and the function needs to be changed to return
-    mod = PyPegen_ASTFromString(str, arena);
+    mod = PyPegen_ASTFromString(str, filename, arena);
 
     mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena);
     if (mod != NULL)

Tools/peg_generator/peg_parser/pegen.c

@@ -533,8 +533,8 @@ Parser_New(struct tok_state *tok, mod_ty (*parse_func)(Parser *), int input_mode
     return p;
 }
 
-void *
-run_parser(Parser *p, int mode)
+mod_ty
+run_parser(Parser *p)
 {
     int error = setjmp(p->error_env);
     if (error) {
@@ -556,89 +556,65 @@ run_parser(Parser *p, int mode)
         return NULL;
     }
 
-    void *result = NULL;
-    if (mode == CODE_OBJECT) {
-        result = PyAST_CompileObject(res, p->tok->filename, NULL, -1, p->arena);
-    }
-    else if (mode == AST_OBJECT) {
-        result = PyAST_mod2obj(res);
-    }
-    else {
-        result = res;
-    }
-
-    return result;
+    return res;
 }
 
-void *
-run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), int mode,
-                     PyArena *arena)
+mod_ty
+run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena)
 {
     FILE *fp = fopen(filename, "rb");
     if (fp == NULL) {
         PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename);
         return NULL;
     }
 
-    PyObject *filename_ob = NULL;
-    if ((filename_ob = PyUnicode_FromString(filename)) == NULL) {
-        return NULL;
-    }
-
     // From here on we need to clean up even if there's an error
-    void *result = NULL;
+    mod_ty result = NULL;
 
     struct tok_state *tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
     if (tok == NULL) {
         goto error;
     }
-
-    // Transfers ownership
     tok->filename = filename_ob;
-    filename_ob = NULL;
+    Py_INCREF(filename_ob);
 
     Parser *p = Parser_New(tok, parse_func, FILE_INPUT, arena);
     if (p == NULL) {
-        PyTokenizer_Free(tok);
-        goto error;
+        goto after_tok_error;
     }
 
-    result = run_parser(p, mode);
-
+    result = run_parser(p);
     Parser_Free(p);
-    PyTokenizer_Free(tok);
 
+after_tok_error:
+    PyTokenizer_Free(tok);
 error:
     fclose(fp);
-    Py_XDECREF(filename_ob);
     return result;
 }
 
-void *
-run_parser_from_string(const char *str, mod_ty (*parse_func)(Parser *), int mode,
-                       PyArena *arena)
+mod_ty
+run_parser_from_string(const char *str, mod_ty(*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena)
 {
-    void *result = NULL;
     struct tok_state *tok = PyTokenizer_FromString(str, 1);
     if (tok == NULL) {
         return NULL;
     }
+    tok->filename = filename_ob;
+    Py_INCREF(filename_ob);
 
-    tok->filename = PyUnicode_FromString("<string>");
-    if (tok->filename == NULL) {
-        PyTokenizer_Free(tok);
-        return NULL;
-    }
+    // We need to clear up from here on
+    mod_ty result = NULL;
 
     Parser *p = Parser_New(tok, parse_func, STRING_INPUT, arena);
     if (p == NULL) {
-        PyTokenizer_Free(tok);
-        return NULL;
+        goto error;
     }
 
-    result = run_parser(p, mode);
-
+    result = run_parser(p);
     Parser_Free(p);
+
+error:
     PyTokenizer_Free(tok);
     return result;
 }