Skip to content

gh-105481: Generate the opcode lists in dis from data extracted from bytecodes.c #106758

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
iritkatriel merged 30 commits into from
Jul 18, 2023
Merged

gh-105481: Generate the opcode lists in dis from data extracted from bytecodes.c #106758

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
36a7d56
gh-105481: add haslocal to _opcode.py. Generate most oplists in opcod…
iritkatriel Jul 14, 2023
75019ca
added HAS_FREE flag. Removed HAS_FREE from the HAS_LOCAL list
iritkatriel Jul 14, 2023
494680a
add hasjump, soft deprecate hasjrel and hasjabs. Fix build bug
iritkatriel Jul 15, 2023
24786fb
generate hasexc from the C macros instead of defining it again
iritkatriel Jul 15, 2023
563b431
typo
iritkatriel Jul 15, 2023
227756f
Merge remote-tracking branch 'upstream/main' into opcode_py
iritkatriel Jul 17, 2023
1b13b96
ascii quotes
iritkatriel Jul 17, 2023
0f5ae32
make clinic
iritkatriel Jul 17, 2023
289fad1
remove oplists
iritkatriel Jul 17, 2023
598ba2a
import less from opcode in build script. _opcode.has_* don't exist pr…
iritkatriel Jul 17, 2023
d2d355e
hascompare in old versions as well
iritkatriel Jul 17, 2023
27eb2cc
remove redundant init
iritkatriel Jul 17, 2023
998024a
📜🤖 Added by blurb_it.
blurb-it[bot] Jul 17, 2023
ffa4ee3
Merge branch 'main' into opcode_py
iritkatriel Jul 17, 2023
ce76a60
remove unnecessary comment
iritkatriel Jul 17, 2023
4194a12
make the tests more precise
iritkatriel Jul 17, 2023
6b8c46b
sort the lists
iritkatriel Jul 17, 2023
e15f7d5
address some of the code review comments
iritkatriel Jul 18, 2023
21fe0a4
remove hard-coded lists from the tests
iritkatriel Jul 18, 2023
75795db
remove opcode metadata file from generated files list so diff are vis…
iritkatriel Jul 18, 2023
1a0a2b2
import _opcode directly into dis, rather than via opcode
iritkatriel Jul 18, 2023
f530469
add stack effect to dis.__all__
iritkatriel Jul 18, 2023
71ffe34
Merge branch 'main' into opcode_py
iritkatriel Jul 18, 2023
d525c53
import _specializations, _specialized_instructions directly from _opc…
iritkatriel Jul 18, 2023
5e1c4a4
update Tools/scripts/summarize_stats.py to use _opcode_metadata inste…
iritkatriel Jul 18, 2023
6077263
Revert "update Tools/scripts/summarize_stats.py to use _opcode_metada…
iritkatriel Jul 18, 2023
98ba11f
Revert "import _specializations, _specialized_instructions directly f…
iritkatriel Jul 18, 2023
cc8e5e1
Revert "add stack effect to dis.__all__"
iritkatriel Jul 18, 2023
31dbfec
Revert "import _opcode directly into dis, rather than via opcode"
iritkatriel Jul 18, 2023
1abf6ca
Merge branch 'main' into opcode_py
iritkatriel Jul 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 21 additions & 7 deletions Doc/library/dis.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1803,15 +1803,12 @@ instructions:
Sequence of bytecodes that access an attribute by name.


.. data:: hasjrel

Sequence of bytecodes that have a relative jump target.
.. data:: hasjump

Sequence of bytecodes that have a jump target. All jumps
are relative.

.. data:: hasjabs

Sequence of bytecodes that have an absolute jump target.

.. versionadded:: 3.13

.. data:: haslocal

Expand All @@ -1827,3 +1824,20 @@ instructions:
Sequence of bytecodes that set an exception handler.

.. versionadded:: 3.12


.. data:: hasjrel

Sequence of bytecodes that have a relative jump target.

.. deprecated:: 3.13
All jumps are now relative. Use :data:`hasjump`.


.. data:: hasjabs

Sequence of bytecodes that have an absolute jump target.

.. deprecated:: 3.13
All jumps are now relative. This list is empty.

5 changes: 4 additions & 1 deletion Include/cpython/compile.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ PyAPI_FUNC(int) PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int j
PyAPI_FUNC(int) PyUnstable_OpcodeIsValid(int opcode);
PyAPI_FUNC(int) PyUnstable_OpcodeHasArg(int opcode);
PyAPI_FUNC(int) PyUnstable_OpcodeHasConst(int opcode);
PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode);
PyAPI_FUNC(int) PyUnstable_OpcodeHasJump(int opcode);
PyAPI_FUNC(int) PyUnstable_OpcodeHasFree(int opcode);
PyAPI_FUNC(int) PyUnstable_OpcodeHasLocal(int opcode);
PyAPI_FUNC(int) PyUnstable_OpcodeHasName(int opcode);
PyAPI_FUNC(int) PyUnstable_OpcodeHasExc(int opcode);

14 changes: 8 additions & 6 deletions Include/internal/pycore_opcode_metadata.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -952,11 +952,13 @@ enum InstructionFormat { INSTR_FMT_IB, INSTR_FMT_IBC, INSTR_FMT_IBC00, INSTR_FMT
#define HAS_CONST_FLAG (2)
#define HAS_NAME_FLAG (4)
#define HAS_JUMP_FLAG (8)
#define HAS_LOCAL_FLAG (16)
#define HAS_FREE_FLAG (16)
#define HAS_LOCAL_FLAG (32)
#define OPCODE_HAS_ARG(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_ARG_FLAG))
#define OPCODE_HAS_CONST(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_CONST_FLAG))
#define OPCODE_HAS_NAME(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_NAME_FLAG))
#define OPCODE_HAS_JUMP(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_JUMP_FLAG))
#define OPCODE_HAS_FREE(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_FREE_FLAG))
#define OPCODE_HAS_LOCAL(OP) (_PyOpcode_opcode_metadata[OP].flags & (HAS_LOCAL_FLAG))

struct opcode_metadata {
Expand Down Expand Up @@ -1080,11 +1082,11 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = {
[LOAD_GLOBAL_MODULE] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG },
[LOAD_GLOBAL_BUILTIN] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG },
[DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
[MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
[DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
[LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
[LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
[STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG },
[MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG },
[DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG },
[LOAD_FROM_DICT_OR_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG },
[LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG },
[STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG },
[COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
[BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG },
Expand Down
40 changes: 17 additions & 23 deletions Lib/opcode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@
operate on bytecodes (e.g. peephole optimizers).
"""

__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs",
"haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap",
"HAVE_ARGUMENT", "EXTENDED_ARG"]
__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjump", "hasjrel",
"hasjabs", "haslocal", "hascompare", "hasfree", "hasexc",
"opname", "opmap", "HAVE_ARGUMENT", "EXTENDED_ARG"]

# It's a chicken-and-egg I'm afraid:
# We're imported before _opcode's made.
Expand All @@ -30,16 +30,14 @@
cmp_op = ('<', '<=', '==', '!=', '>', '>=')

hascompare = []
hasfree = []
hasexc = []


ENABLE_SPECIALIZATION = True

def is_pseudo(op):
return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE

oplists = [hascompare, hasfree, hasexc]
oplists = [hascompare]

opmap = {}

Expand Down Expand Up @@ -174,13 +172,9 @@ def pseudo_op(name, op, real_ops):
def_op('BUILD_SLICE', 133) # Number of items
def_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards)
def_op('MAKE_CELL', 135)
hasfree.append(135)
def_op('LOAD_DEREF', 137)
hasfree.append(137)
def_op('STORE_DEREF', 138)
hasfree.append(138)
def_op('DELETE_DEREF', 139)
hasfree.append(139)
def_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards)
def_op('LOAD_SUPER_ATTR', 141)
def_op('CALL_FUNCTION_EX', 142) # Flags
Expand All @@ -190,7 +184,6 @@ def pseudo_op(name, op, real_ops):
def_op('LIST_APPEND', 145)
def_op('SET_ADD', 146)
def_op('MAP_ADD', 147)
hasfree.append(148)
def_op('COPY_FREE_VARS', 149)
def_op('YIELD_VALUE', 150)
def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py
Expand All @@ -214,7 +207,6 @@ def pseudo_op(name, op, real_ops):
def_op('CALL_INTRINSIC_2', 174)
def_op('LOAD_FROM_DICT_OR_GLOBALS', 175)
def_op('LOAD_FROM_DICT_OR_DEREF', 176)
hasfree.append(176)
def_op('SET_FUNCTION_ATTRIBUTE', 177) # Attribute

# Optimizer hook
Expand Down Expand Up @@ -247,11 +239,8 @@ def pseudo_op(name, op, real_ops):
MIN_PSEUDO_OPCODE = 256

pseudo_op('SETUP_FINALLY', 256, ['NOP'])
hasexc.append(256)
pseudo_op('SETUP_CLEANUP', 257, ['NOP'])
hasexc.append(257)
pseudo_op('SETUP_WITH', 258, ['NOP'])
hasexc.append(258)
pseudo_op('POP_BLOCK', 259, ['NOP'])

pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD'])
Expand All @@ -276,20 +265,25 @@ def pseudo_op(name, op, real_ops):
# _opcode may not be ready during early stages of the build
Copy link
Member

@gvanrossum gvanrossum Jul 14, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is that separate from Larry's limerick at the top?

Copy link
Member Author

@iritkatriel iritkatriel Jul 14, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's the same thing. I'm planning to try and get rid of this issue - figure out what is needed in the build and separate it out somehow.

Copy link
Member

@gvanrossum gvanrossum Jul 14, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Bot we've gotta keep a limerick in. :-)

Copy link
Member Author

@iritkatriel iritkatriel Jul 15, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it in the stable ABI?

try:
import _opcode
except ImportError:
hasarg = [op for op in opmap.values() if _opcode.has_arg(op)]
hasconst = [op for op in opmap.values() if _opcode.has_const(op)]
hasname = [op for op in opmap.values() if _opcode.has_name(op)]
hasjump = [op for op in opmap.values() if _opcode.has_jump(op)]
hasjrel = hasjump # for backward compatibility
hasjabs = []
hasfree = [op for op in opmap.values() if _opcode.has_free(op)]
haslocal = [op for op in opmap.values() if _opcode.has_local(op)]
hasexc = [op for op in opmap.values() if _opcode.has_exc(op)]
except (ImportError, AttributeError):
hasarg = []
hasconst = []
hasname = []
hasjump = []
hasjrel = []
hasjabs = []
hasfree = []
haslocal = []
else:
hasarg = [op for op in opmap.values() if _opcode.has_arg(op)]
hasconst = [op for op in opmap.values() if _opcode.has_const(op)]
hasname = [op for op in opmap.values() if _opcode.has_name(op)]
hasjrel = [op for op in opmap.values() if _opcode.has_jump(op)]
hasjabs = []
haslocal = [op for op in opmap.values() if _opcode.has_local(op)]
hasexc = []

_nb_ops = [
("NB_ADD", "+"),
Expand Down
15 changes: 15 additions & 0 deletions Lib/test/test__opcode.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ def test_invalid_opcodes(self):
self.check_bool_function_result(_opcode.has_const, invalid, False)
self.check_bool_function_result(_opcode.has_name, invalid, False)
self.check_bool_function_result(_opcode.has_jump, invalid, False)
self.check_bool_function_result(_opcode.has_free, invalid, False)
self.check_bool_function_result(_opcode.has_local, invalid, False)
self.check_bool_function_result(_opcode.has_exc, invalid, False)

def test_is_valid(self):
names = [
Expand Down Expand Up @@ -62,13 +64,26 @@ def test_has_jump(self):
self.check_bool_function_result(_opcode.has_jump, has_jump, True)
self.check_bool_function_result(_opcode.has_jump, no_jump, False)

def test_has_free(self):
has_free = ['MAKE_CELL', 'LOAD_DEREF', 'STORE_DEREF', 'DELETE_DEREF',
'LOAD_FROM_DICT_OR_DEREF']
no_free = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE']
self.check_bool_function_result(_opcode.has_free, has_free, True)
self.check_bool_function_result(_opcode.has_free, no_free, False)

def test_has_local(self):
has_local = ['LOAD_FAST', 'LOAD_FAST_CHECK', 'LOAD_FAST_AND_CLEAR',
'STORE_FAST_MAYBE_NULL', 'LOAD_CLOSURE']
no_local = ['SETUP_WITH', 'POP_TOP', 'NOP', 'CACHE']
self.check_bool_function_result(_opcode.has_local, has_local, True)
self.check_bool_function_result(_opcode.has_local, no_local, False)

def test_has_exc(self):
has_exc = ['SETUP_FINALLY', 'SETUP_WITH', 'SETUP_CLEANUP']
no_exc = ['DELETE_DEREF', 'POP_TOP', 'NOP', 'CACHE']
self.check_bool_function_result(_opcode.has_exc, has_exc, True)
self.check_bool_function_result(_opcode.has_exc, no_exc, False)

def test_stack_effect(self):
self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1)
self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1)
Expand Down
42 changes: 41 additions & 1 deletion Modules/_opcode.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,29 @@ _opcode_has_jump_impl(PyObject *module, int opcode)

/*[clinic input]

_opcode.has_free -> bool

opcode: int

Return True if the opcode accesses a free variable, False otherwise.

Note that ‘free’ in this context refers to names in the current scope
that are referenced by inner scopes or names in outer scopes that are
referenced from this scope. It does not include references to global
or builtin scopes.
[clinic start generated code]*/

static int
_opcode_has_free_impl(PyObject *module, int opcode)
/*[clinic end generated code: output=d81ae4d79af0ee26 input=f94ce9c2475f3ff0]*/
{
return PyUnstable_OpcodeIsValid(opcode) &&
PyUnstable_OpcodeHasFree(opcode);

}

/*[clinic input]

_opcode.has_local -> bool

opcode: int
Expand All @@ -160,9 +183,24 @@ _opcode_has_local_impl(PyObject *module, int opcode)
{
return PyUnstable_OpcodeIsValid(opcode) &&
PyUnstable_OpcodeHasLocal(opcode);

}

/*[clinic input]

_opcode.has_exc -> bool

opcode: int

Return True if the opcode sets and exception handler, False otherwise.
[clinic start generated code]*/

static int
_opcode_has_exc_impl(PyObject *module, int opcode)
/*[clinic end generated code: output=41b68dff0ec82a52 input=a3f202bb94f233b5]*/
{
return PyUnstable_OpcodeIsValid(opcode) &&
PyUnstable_OpcodeHasExc(opcode);
}

/*[clinic input]

Expand Down Expand Up @@ -190,7 +228,9 @@ opcode_functions[] = {
_OPCODE_HAS_CONST_METHODDEF
_OPCODE_HAS_NAME_METHODDEF
_OPCODE_HAS_JUMP_METHODDEF
_OPCODE_HAS_FREE_METHODDEF
_OPCODE_HAS_LOCAL_METHODDEF
_OPCODE_HAS_EXC_METHODDEF
_OPCODE_GET_SPECIALIZATION_STATS_METHODDEF
{NULL, NULL, 0, NULL}
};
Expand Down
Loading