From 128f5fbcbc99a524dcb10fffa0172365ca38839e Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 23 Mar 2020 22:19:46 +0200 Subject: [PATCH 1/4] Rewrite run_parser* functions to always return a mod_ty * The caller is responsible for transforming the mod_ty to a code object or an AST object --- Grammar/python.gram | 2 + Include/pegen_interface.h | 8 +-- Modules/peg_parser.c | 32 +++++++++--- Parser/pegen/parse.c | 2 + Parser/pegen/peg_api.c | 26 +++++++--- Parser/pegen/pegen.c | 62 ++++++++---------------- Parser/pegen/pegen.h | 11 +---- Python/pythonrun.c | 2 +- Tools/peg_generator/peg_parser/pegen.c | 62 ++++++++---------------- Tools/peg_generator/peg_parser/pegen.h | 11 +---- Tools/peg_generator/pegen/c_generator.py | 60 ++++++++++++++++++----- 11 files changed, 145 insertions(+), 133 deletions(-) diff --git a/Grammar/python.gram b/Grammar/python.gram index 7c7366c7a212fa..b5920e9d6d4997 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -8,6 +8,8 @@ parse_start(Parser *p) { return start_rule(p); } + +// The end ''' start[mod_ty]: a=[statements] ENDMARKER { Module(a, NULL, p->arena) } diff --git a/Include/pegen_interface.h b/Include/pegen_interface.h index c48137fff04619..4762c8e1667c39 100644 --- a/Include/pegen_interface.h +++ b/Include/pegen_interface.h @@ -8,10 +8,10 @@ extern "C" { #include "Python.h" #include "Python-ast.h" -PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, PyArena *arena); -PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(const char *str, PyArena *arena); -PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, PyArena *arena); -PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromString(const char *str, PyArena *arena); +PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, PyObject *filename_ob, PyArena *arena); +PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(const char *str, PyObject *filename_ob, PyArena *arena); +PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, PyObject *filename_ob, PyArena *arena); +PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromString(const char *str, PyObject *filename_ob, PyArena *arena); #ifdef __cplusplus } diff --git a/Modules/peg_parser.c b/Modules/peg_parser.c index b0122fb6e3595b..54d5d8f72bb07b 100644 --- a/Modules/peg_parser.c +++ b/Modules/peg_parser.c @@ -15,13 +15,21 @@ _Py_parse_file(PyObject *self, PyObject *args) return NULL; } - mod_ty res = PyPegen_ASTFromFile(filename, arena); + PyObject *result = NULL; + + PyObject *filename_ob = PyUnicode_FromString(filename); + if (filename_ob == NULL) { + goto error; + } + + mod_ty res = PyPegen_ASTFromFile(filename, filename_ob, arena); if (res == NULL) { - PyArena_Free(arena); - return NULL; + goto error; } - PyObject *result = PyAST_mod2obj(res); + result = PyAST_mod2obj(res); +error: + Py_XDECREF(filename_ob); PyArena_Free(arena); return result; } @@ -40,13 +48,21 @@ _Py_parse_string(PyObject *self, PyObject *args) return NULL; } - mod_ty res = PyPegen_ASTFromString(the_string, arena); + PyObject *result = NULL; + + PyObject *filename_ob = PyUnicode_FromString(""); + if (filename_ob == NULL) { + goto error; + } + + mod_ty res = PyPegen_ASTFromString(the_string, filename_ob, arena); if (res == NULL) { - PyArena_Free(arena); - return NULL; + goto error; } - PyObject *result = PyAST_mod2obj(res); + result = PyAST_mod2obj(res); +error: + Py_XDECREF(filename_ob); PyArena_Free(arena); return result; } diff --git a/Parser/pegen/parse.c b/Parser/pegen/parse.c index de7d64c2a1e38c..05bf0fe9347992 100644 --- a/Parser/pegen/parse.c +++ b/Parser/pegen/parse.c @@ -13564,3 +13564,5 @@ parse_start(Parser *p) { return start_rule(p); } + +// The end diff --git a/Parser/pegen/peg_api.c b/Parser/pegen/peg_api.c index 479028c37d4c61..8f5558f55084ae 100644 --- a/Parser/pegen/peg_api.c +++ b/Parser/pegen/peg_api.c @@ -4,25 +4,35 @@ #include "pegen.h" mod_ty -PyPegen_ASTFromString(const char *str, PyArena *arena) +PyPegen_ASTFromString(const char *str, PyObject *filename_ob, PyArena *arena) { - return run_parser_from_string(str, parse_start, RAW_AST_OBJECT, arena); + return run_parser_from_string(str, parse_start, filename_ob, arena); } mod_ty -PyPegen_ASTFromFile(const char *filename, PyArena *arena) +PyPegen_ASTFromFile(const char *filename, PyObject *filename_ob, PyArena *arena) { - return run_parser_from_file(filename, parse_start, RAW_AST_OBJECT, arena); + return run_parser_from_file(filename, parse_start, filename_ob, arena); } PyCodeObject * -PyPegen_CodeObjectFromString(const char *str, PyArena *arena) +PyPegen_CodeObjectFromString(const char *str, PyObject *filename_ob, PyArena *arena) { - return run_parser_from_string(str, parse_start, CODE_OBJECT, arena); + mod_ty res = run_parser_from_string(str, parse_start, filename_ob, arena); + if (res == NULL) { + return NULL; + } + + return PyAST_CompileObject(res, filename_ob, NULL, -1, arena); } PyCodeObject * -PyPegen_CodeObjectFromFile(const char *file, PyArena *arena) +PyPegen_CodeObjectFromFile(const char *filename, PyObject *filename_ob, PyArena *arena) { - return run_parser_from_file(file, parse_start, CODE_OBJECT, arena); + mod_ty res = run_parser_from_string(filename, parse_start, filename_ob, arena); + if (res == NULL) { + return NULL; + } + + return PyAST_CompileObject(res, filename_ob, NULL, -1, arena); } diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index f76518d59fb070..7116d668e7c0a8 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -536,8 +536,8 @@ Parser_New(struct tok_state *tok, mod_ty(*parse_func)(Parser *), int input_mode, return p; } -void * -run_parser(Parser *p, int mode) +mod_ty +run_parser(Parser *p) { int error = setjmp(p->error_env); if (error) { @@ -559,20 +559,11 @@ run_parser(Parser *p, int mode) return NULL; } - void *result = NULL; - if (mode == CODE_OBJECT) { - result = PyAST_CompileObject(res, p->tok->filename, NULL, -1, p->arena); - } else if (mode == AST_OBJECT) { - result = PyAST_mod2obj(res); - } else { - result = res; - } - - return result; + return res; } -void * -run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), int mode, PyArena *arena) +mod_ty +run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena) { FILE *fp = fopen(filename, "rb"); if (fp == NULL) { @@ -580,64 +571,53 @@ run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), int m return NULL; } - PyObject *filename_ob = NULL; - if ((filename_ob = PyUnicode_FromString(filename)) == NULL) { - return NULL; - } - // From here on we need to clean up even if there's an error - void *result = NULL; + mod_ty result = NULL; struct tok_state *tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL); if (tok == NULL) { goto error; } - - // Transfers ownership tok->filename = filename_ob; - filename_ob = NULL; + Py_INCREF(filename_ob); Parser *p = Parser_New(tok, parse_func, FILE_INPUT, arena); if (p == NULL) { - PyTokenizer_Free(tok); - goto error; + goto after_tok_error; } - result = run_parser(p, mode); - + result = run_parser(p); Parser_Free(p); - PyTokenizer_Free(tok); +after_tok_error: + PyTokenizer_Free(tok); error: fclose(fp); - Py_XDECREF(filename_ob); return result; } -void * -run_parser_from_string(const char *str, mod_ty(*parse_func)(Parser *), int mode, PyArena *arena) +mod_ty +run_parser_from_string(const char *str, mod_ty(*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena) { - void *result = NULL; struct tok_state *tok = PyTokenizer_FromString(str, 1); if (tok == NULL) { return NULL; } + tok->filename = filename_ob; + Py_INCREF(filename_ob); - tok->filename = PyUnicode_FromString(""); - if (tok->filename == NULL) { - PyTokenizer_Free(tok); - return NULL; - } + // We need to clear up from here on + mod_ty result = NULL; Parser *p = Parser_New(tok, parse_func, STRING_INPUT, arena); if (p == NULL) { - PyTokenizer_Free(tok); - return NULL; + goto error; } - result = run_parser(p, mode); - + result = run_parser(p); Parser_Free(p); + +error: PyTokenizer_Free(tok); return result; } diff --git a/Parser/pegen/pegen.h b/Parser/pegen/pegen.h index 7ba70f9731219d..b33c332a316e45 100644 --- a/Parser/pegen/pegen.h +++ b/Parser/pegen/pegen.h @@ -14,13 +14,6 @@ enum INPUT_MODE { }; typedef enum INPUT_MODE INPUT_MODE; -enum MODE { - RAW_AST_OBJECT, - AST_OBJECT, - CODE_OBJECT -}; -typedef enum MODE MODE; - typedef struct _memo { int type; void *node; @@ -143,8 +136,8 @@ CHECK_CALL_NULL_ALLOWED(Parser *p, void *result) #define CHECK_NULL_ALLOWED(result) CHECK_CALL_NULL_ALLOWED(p, result) PyObject *new_identifier(Parser *, char *); -void *run_parser_from_file(const char *, mod_ty(*)(Parser *), int, PyArena *); -void *run_parser_from_string(const char *, mod_ty(*)(Parser *), int, PyArena *); +mod_ty run_parser_from_file(const char *, mod_ty(*)(Parser *), PyObject *, PyArena *); +mod_ty run_parser_from_string(const char *, mod_ty(*)(Parser *), PyObject *, PyArena *); asdl_seq *singleton_seq(Parser *, void *); asdl_seq *seq_insert_in_front(Parser *, void *, asdl_seq *); asdl_seq *seq_flatten(Parser *, asdl_seq *); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index dce601729036cc..5a00616daff002 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1036,7 +1036,7 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals, // I left this here so when you run 'nm python | grep PyPegen' you can see // that the symbol is included and this function is called. Obviously this is // wrong, and the function needs to be changed to return - mod = PyPegen_ASTFromString(str, arena); + mod = PyPegen_ASTFromString(str, filename, arena); mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); if (mod != NULL) diff --git a/Tools/peg_generator/peg_parser/pegen.c b/Tools/peg_generator/peg_parser/pegen.c index 31e2c150fd957f..2be2b6ef1d351f 100644 --- a/Tools/peg_generator/peg_parser/pegen.c +++ b/Tools/peg_generator/peg_parser/pegen.c @@ -533,8 +533,8 @@ Parser_New(struct tok_state *tok, mod_ty(*parse_func)(Parser *), int input_mode, return p; } -void * -run_parser(Parser *p, int mode) +mod_ty +run_parser(Parser *p) { int error = setjmp(p->error_env); if (error) { @@ -556,20 +556,11 @@ run_parser(Parser *p, int mode) return NULL; } - void *result = NULL; - if (mode == CODE_OBJECT) { - result = PyAST_CompileObject(res, p->tok->filename, NULL, -1, p->arena); - } else if (mode == AST_OBJECT) { - result = PyAST_mod2obj(res); - } else { - result = res; - } - - return result; + return res; } -void * -run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), int mode, PyArena *arena) +mod_ty +run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena) { FILE *fp = fopen(filename, "rb"); if (fp == NULL) { @@ -577,64 +568,53 @@ run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), int m return NULL; } - PyObject *filename_ob = NULL; - if ((filename_ob = PyUnicode_FromString(filename)) == NULL) { - return NULL; - } - // From here on we need to clean up even if there's an error - void *result = NULL; + mod_ty result = NULL; struct tok_state *tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL); if (tok == NULL) { goto error; } - - // Transfers ownership tok->filename = filename_ob; - filename_ob = NULL; + Py_INCREF(filename_ob); Parser *p = Parser_New(tok, parse_func, FILE_INPUT, arena); if (p == NULL) { - PyTokenizer_Free(tok); - goto error; + goto after_tok_error; } - result = run_parser(p, mode); - + result = run_parser(p); Parser_Free(p); - PyTokenizer_Free(tok); +after_tok_error: + PyTokenizer_Free(tok); error: fclose(fp); - Py_XDECREF(filename_ob); return result; } -void * -run_parser_from_string(const char *str, mod_ty(*parse_func)(Parser *), int mode, PyArena *arena) +mod_ty +run_parser_from_string(const char *str, mod_ty(*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena) { - void *result = NULL; struct tok_state *tok = PyTokenizer_FromString(str, 1); if (tok == NULL) { return NULL; } + tok->filename = filename_ob; + Py_INCREF(filename_ob); - tok->filename = PyUnicode_FromString(""); - if (tok->filename == NULL) { - PyTokenizer_Free(tok); - return NULL; - } + // We need to clear up from here on + mod_ty result = NULL; Parser *p = Parser_New(tok, parse_func, STRING_INPUT, arena); if (p == NULL) { - PyTokenizer_Free(tok); - return NULL; + goto error; } - result = run_parser(p, mode); - + result = run_parser(p); Parser_Free(p); + +error: PyTokenizer_Free(tok); return result; } diff --git a/Tools/peg_generator/peg_parser/pegen.h b/Tools/peg_generator/peg_parser/pegen.h index 32c770ca978dae..0c9b4639039191 100644 --- a/Tools/peg_generator/peg_parser/pegen.h +++ b/Tools/peg_generator/peg_parser/pegen.h @@ -14,13 +14,6 @@ enum INPUT_MODE { }; typedef enum INPUT_MODE INPUT_MODE; -enum MODE { - RAW_AST_OBJECT, - AST_OBJECT, - CODE_OBJECT, -}; -typedef enum MODE MODE; - typedef struct _memo { int type; void *node; @@ -143,8 +136,8 @@ CHECK_CALL_NULL_ALLOWED(Parser *p, void *result) #define CHECK_NULL_ALLOWED(result) CHECK_CALL_NULL_ALLOWED(p, result) PyObject *new_identifier(Parser *, char *); -void *run_parser_from_file(const char *, mod_ty(*)(Parser *), int, PyArena *); -void *run_parser_from_string(const char *, mod_ty(*)(Parser *), int, PyArena *); +mod_ty run_parser_from_file(const char *, mod_ty(*)(Parser *), PyObject *, PyArena *); +mod_ty run_parser_from_string(const char *, mod_ty(*)(Parser *), PyObject *, PyArena *); asdl_seq *singleton_seq(Parser *, void *); asdl_seq *seq_insert_in_front(Parser *, void *, asdl_seq *); asdl_seq *seq_flatten(Parser *, asdl_seq *); diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 14e1aeb9838feb..c0495276d5cfe2 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -30,28 +30,53 @@ """ EXTENSION_SUFFIX = """ +PyObject * +_build_return_object(mod_ty module, int mode, PyObject *filename_ob, PyArena *arena) +{ + if (mode == 2) { + return (PyObject *)PyAST_CompileObject(module, filename_ob, NULL, -1, arena); + } else if (mode == 1) { + return PyAST_mod2obj(module); + } else { + Py_INCREF(Py_None); + return Py_None; + } +} + static PyObject * parse_file(PyObject *self, PyObject *args, PyObject *kwds) { static char *keywords[] = {"file", "mode", NULL}; const char *filename; int mode = %(mode)s; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", keywords, &filename, &mode)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", keywords, &filename, &mode)) { return NULL; - if (mode < 0 || mode > %(mode)s) + } + if (mode < 0 || mode > %(mode)s) { return PyErr_Format(PyExc_ValueError, "Bad mode, must be 0 <= mode <= %(mode)s"); + } PyArena *arena = PyArena_New(); if (arena == NULL) { return NULL; } - void *result = run_parser_from_file(filename, start_rule, mode, arena); - if (result != NULL && mode == 0) { - result = Py_None; - Py_INCREF(result); + PyObject *result = NULL; + + PyObject *filename_ob = PyUnicode_FromString(filename); + if (filename_ob == NULL) { + goto error; } + mod_ty res = run_parser_from_file(filename, start_rule, filename_ob, arena); + if (res == NULL) { + goto error; + } + + result = _build_return_object(res, mode, filename_ob, arena); + +error: + Py_XDECREF(filename_ob); PyArena_Free(arena); return result; } @@ -62,22 +87,33 @@ static char *keywords[] = {"str", "mode", NULL}; const char *the_string; int mode = %(mode)s; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", keywords, &the_string, &mode)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|i", keywords, &the_string, &mode)) { return NULL; - if (mode < 0 || mode > %(mode)s) + } + if (mode < 0 || mode > %(mode)s) { return PyErr_Format(PyExc_ValueError, "Bad mode, must be 0 <= mode <= %(mode)s"); + } PyArena *arena = PyArena_New(); if (arena == NULL) { return NULL; } - void *result = run_parser_from_string(the_string, start_rule, mode, arena); - if (result != NULL && mode == 0) { - result = Py_None; - Py_INCREF(result); + PyObject *result = NULL; + + PyObject *filename_ob = PyUnicode_FromString(""); + if (filename_ob == NULL) { + goto error; + } + + mod_ty res = run_parser_from_string(the_string, start_rule, filename_ob, arena); + if (res == NULL) { + goto error; } + result = _build_return_object(res, mode, filename_ob, arena); +error: + Py_XDECREF(filename_ob); PyArena_Free(arena); return result; } From 85e6adf8eb6d4d81a109479808b5d196ec4efb71 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 23 Mar 2020 23:08:51 +0200 Subject: [PATCH 2/4] Run clang-format --- Parser/pegen/pegen.c | 6 ++++-- Tools/peg_generator/peg_parser/pegen.c | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index b152f7ce5f7c8a..48514b6213f507 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -563,7 +563,8 @@ run_parser(Parser *p) } mod_ty -run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena) +run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), + PyObject *filename_ob, PyArena *arena) { FILE *fp = fopen(filename, "rb"); if (fp == NULL) { @@ -597,7 +598,8 @@ run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), PyObj } mod_ty -run_parser_from_string(const char *str, mod_ty(*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena) +run_parser_from_string(const char *str, mod_ty (*parse_func)(Parser *), PyObject *filename_ob, + PyArena *arena) { struct tok_state *tok = PyTokenizer_FromString(str, 1); if (tok == NULL) { diff --git a/Tools/peg_generator/peg_parser/pegen.c b/Tools/peg_generator/peg_parser/pegen.c index 268e748f609757..ba91fe73f4cedc 100644 --- a/Tools/peg_generator/peg_parser/pegen.c +++ b/Tools/peg_generator/peg_parser/pegen.c @@ -560,7 +560,8 @@ run_parser(Parser *p) } mod_ty -run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena) +run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), + PyObject *filename_ob, PyArena *arena) { FILE *fp = fopen(filename, "rb"); if (fp == NULL) { @@ -594,7 +595,8 @@ run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), PyObj } mod_ty -run_parser_from_string(const char *str, mod_ty(*parse_func)(Parser *), PyObject *filename_ob, PyArena *arena) +run_parser_from_string(const char *str, mod_ty (*parse_func)(Parser *), PyObject *filename_ob, + PyArena *arena) { struct tok_state *tok = PyTokenizer_FromString(str, 1); if (tok == NULL) { From cbbd5906f856eb57850194f768816e5ce7e27db5 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 23 Mar 2020 23:41:00 +0200 Subject: [PATCH 3/4] Respond to review comments --- Parser/pegen/peg_api.c | 4 ++-- Parser/pegen/pegen.c | 2 ++ Tools/peg_generator/peg_parser/pegen.c | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Parser/pegen/peg_api.c b/Parser/pegen/peg_api.c index 8f5558f55084ae..b13dd229332190 100644 --- a/Parser/pegen/peg_api.c +++ b/Parser/pegen/peg_api.c @@ -18,7 +18,7 @@ PyPegen_ASTFromFile(const char *filename, PyObject *filename_ob, PyArena *arena) PyCodeObject * PyPegen_CodeObjectFromString(const char *str, PyObject *filename_ob, PyArena *arena) { - mod_ty res = run_parser_from_string(str, parse_start, filename_ob, arena); + mod_ty res = PyPegen_ASTFromString(str, filename_ob, arena); if (res == NULL) { return NULL; } @@ -29,7 +29,7 @@ PyPegen_CodeObjectFromString(const char *str, PyObject *filename_ob, PyArena *ar PyCodeObject * PyPegen_CodeObjectFromFile(const char *filename, PyObject *filename_ob, PyArena *arena) { - mod_ty res = run_parser_from_string(filename, parse_start, filename_ob, arena); + mod_ty res = PyPegen_ASTFromFile(filename, filename_ob, arena); if (res == NULL) { return NULL; } diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index 48514b6213f507..74f8e568a5a67f 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -579,6 +579,7 @@ run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), if (tok == NULL) { goto error; } + // This transfers the ownership to the tokenizer tok->filename = filename_ob; Py_INCREF(filename_ob); @@ -605,6 +606,7 @@ run_parser_from_string(const char *str, mod_ty (*parse_func)(Parser *), PyObject if (tok == NULL) { return NULL; } + // This transfers the ownership to the tokenizer tok->filename = filename_ob; Py_INCREF(filename_ob); diff --git a/Tools/peg_generator/peg_parser/pegen.c b/Tools/peg_generator/peg_parser/pegen.c index ba91fe73f4cedc..ce9ed2eebef9b0 100644 --- a/Tools/peg_generator/peg_parser/pegen.c +++ b/Tools/peg_generator/peg_parser/pegen.c @@ -576,6 +576,7 @@ run_parser_from_file(const char *filename, mod_ty (*parse_func)(Parser *), if (tok == NULL) { goto error; } + // This transfers the ownership to the tokenizer tok->filename = filename_ob; Py_INCREF(filename_ob); @@ -602,6 +603,7 @@ run_parser_from_string(const char *str, mod_ty (*parse_func)(Parser *), PyObject if (tok == NULL) { return NULL; } + // This transfers the ownership to the tokenizer tok->filename = filename_ob; Py_INCREF(filename_ob); From 18fe60091164e003fad33a12b4835182c692b715 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 24 Mar 2020 00:21:03 +0200 Subject: [PATCH 4/4] Don't expect the filename and the arena (where possible) objects as an argument --- Include/pegen_interface.h | 8 ++--- Modules/peg_parser.c | 16 ++------- Parser/pegen/peg_api.c | 72 ++++++++++++++++++++++++++++++++------- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/Include/pegen_interface.h b/Include/pegen_interface.h index 4762c8e1667c39..3a46b72d29a787 100644 --- a/Include/pegen_interface.h +++ b/Include/pegen_interface.h @@ -8,10 +8,10 @@ extern "C" { #include "Python.h" #include "Python-ast.h" -PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, PyObject *filename_ob, PyArena *arena); -PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(const char *str, PyObject *filename_ob, PyArena *arena); -PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename, PyObject *filename_ob, PyArena *arena); -PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromString(const char *str, PyObject *filename_ob, PyArena *arena); +PyAPI_FUNC(mod_ty) PyPegen_ASTFromFile(const char *filename, PyArena *arena); +PyAPI_FUNC(mod_ty) PyPegen_ASTFromString(const char *str, PyArena *arena); +PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromFile(const char *filename); +PyAPI_FUNC(PyCodeObject *) PyPegen_CodeObjectFromString(const char *str); #ifdef __cplusplus } diff --git a/Modules/peg_parser.c b/Modules/peg_parser.c index 74f90aa82a4138..a6e01c369a9456 100644 --- a/Modules/peg_parser.c +++ b/Modules/peg_parser.c @@ -17,19 +17,13 @@ _Py_parse_file(PyObject *self, PyObject *args) PyObject *result = NULL; - PyObject *filename_ob = PyUnicode_FromString(filename); - if (filename_ob == NULL) { - goto error; - } - - mod_ty res = PyPegen_ASTFromFile(filename, filename_ob, arena); + mod_ty res = PyPegen_ASTFromFile(filename, arena); if (res == NULL) { goto error; } result = PyAST_mod2obj(res); error: - Py_XDECREF(filename_ob); PyArena_Free(arena); return result; } @@ -50,19 +44,13 @@ _Py_parse_string(PyObject *self, PyObject *args) PyObject *result = NULL; - PyObject *filename_ob = PyUnicode_FromString(""); - if (filename_ob == NULL) { - goto error; - } - - mod_ty res = PyPegen_ASTFromString(the_string, filename_ob, arena); + mod_ty res = PyPegen_ASTFromString(the_string, arena); if (res == NULL) { goto error; } result = PyAST_mod2obj(res); error: - Py_XDECREF(filename_ob); PyArena_Free(arena); return result; } diff --git a/Parser/pegen/peg_api.c b/Parser/pegen/peg_api.c index b13dd229332190..5ed5cd4a9c0cbd 100644 --- a/Parser/pegen/peg_api.c +++ b/Parser/pegen/peg_api.c @@ -4,35 +4,83 @@ #include "pegen.h" mod_ty -PyPegen_ASTFromString(const char *str, PyObject *filename_ob, PyArena *arena) +PyPegen_ASTFromString(const char *str, PyArena *arena) { - return run_parser_from_string(str, parse_start, filename_ob, arena); + PyObject *filename_ob = PyUnicode_FromString(""); + if (filename_ob == NULL) { + return NULL; + } + + mod_ty result = run_parser_from_string(str, parse_start, filename_ob, arena); + Py_XDECREF(filename_ob); + return result; } mod_ty -PyPegen_ASTFromFile(const char *filename, PyObject *filename_ob, PyArena *arena) +PyPegen_ASTFromFile(const char *filename, PyArena *arena) { - return run_parser_from_file(filename, parse_start, filename_ob, arena); + PyObject *filename_ob = PyUnicode_FromString(filename); + if (filename_ob == NULL) { + return NULL; + } + + mod_ty result = run_parser_from_file(filename, parse_start, filename_ob, arena); + Py_XDECREF(filename_ob); + return result; } PyCodeObject * -PyPegen_CodeObjectFromString(const char *str, PyObject *filename_ob, PyArena *arena) +PyPegen_CodeObjectFromString(const char *str) { - mod_ty res = PyPegen_ASTFromString(str, filename_ob, arena); - if (res == NULL) { + PyArena *arena = PyArena_New(); + if (arena == NULL) { return NULL; } - return PyAST_CompileObject(res, filename_ob, NULL, -1, arena); + PyCodeObject *result = NULL; + + PyObject *filename_ob = PyUnicode_FromString(""); + if (filename_ob == NULL) { + goto error; + } + + mod_ty res = PyPegen_ASTFromString(str, arena); + if (res == NULL) { + goto error; + } + + result = PyAST_CompileObject(res, filename_ob, NULL, -1, arena); + +error: + Py_XDECREF(filename_ob); + PyArena_Free(arena); + return result; } PyCodeObject * -PyPegen_CodeObjectFromFile(const char *filename, PyObject *filename_ob, PyArena *arena) +PyPegen_CodeObjectFromFile(const char *filename) { - mod_ty res = PyPegen_ASTFromFile(filename, filename_ob, arena); - if (res == NULL) { + PyArena *arena = PyArena_New(); + if (arena == NULL) { return NULL; } - return PyAST_CompileObject(res, filename_ob, NULL, -1, arena); + PyCodeObject *result = NULL; + + PyObject *filename_ob = PyUnicode_FromString(filename); + if (filename_ob == NULL) { + goto error; + } + + mod_ty res = PyPegen_ASTFromFile(filename, arena); + if (res == NULL) { + goto error; + } + + result = PyAST_CompileObject(res, filename_ob, NULL, -1, arena); + +error: + Py_XDECREF(filename_ob); + PyArena_Free(arena); + return result; }