From 4096133546445b3d051305a4e91db971b6c3c202 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sun, 18 Jun 2023 19:30:36 +0100 Subject: [PATCH 1/9] generate _specializations data from bytecodes.c --- Include/internal/pycore_opcode.h | 104 +++++++++++------------ Include/opcode.h | 108 ++++++++++++------------ Lib/_opcode_metadata.py | 94 +++++++++++++++++++++ Lib/dis.py | 4 +- Lib/importlib/_bootstrap_external.py | 3 +- Lib/opcode.py | 90 -------------------- Lib/test/test_dis.py | 5 +- Makefile.pre.in | 3 +- Python/makeopcodetargets.py | 3 +- Python/opcode_targets.h | 104 +++++++++++------------ Python/stdlib_module_names.h | 1 + Tools/build/generate_opcode_h.py | 28 +++--- Tools/cases_generator/generate_cases.py | 77 ++++++++++++++--- 13 files changed, 343 insertions(+), 281 deletions(-) create mode 100644 Lib/_opcode_metadata.py diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 5935823b7e24ad..6b269e3c02d6c1 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -241,49 +241,49 @@ static const char *const _PyOpcode_OpName[267] = { [INTERPRETER_EXIT] = "INTERPRETER_EXIT", [END_FOR] = "END_FOR", [END_SEND] = "END_SEND", - [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", + [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", - [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", + [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [NOP] = "NOP", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", - [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [UNARY_INVERT] = "UNARY_INVERT", + [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [UNARY_INVERT] = "UNARY_INVERT", + [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", [RESERVED] = "RESERVED", - [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", [BINARY_SUBSCR_LIST_INT] = "BINARY_SUBSCR_LIST_INT", [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", - [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [MAKE_FUNCTION] = "MAKE_FUNCTION", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [SEND_GEN] = "SEND_GEN", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", [FORMAT_SIMPLE] = "FORMAT_SIMPLE", [FORMAT_WITH_SPEC] = "FORMAT_WITH_SPEC", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", - [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", - [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", - [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", - [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", - [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", - [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", + [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", + [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", + [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -291,39 +291,39 @@ static const char *const _PyOpcode_OpName[267] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", - [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", - [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", - [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", + [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", + [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", + [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", + [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [GET_ITER] = "GET_ITER", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [COMPARE_OP_INT] = "COMPARE_OP_INT", [COMPARE_OP_STR] = "COMPARE_OP_STR", + [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", + [RETURN_GENERATOR] = "RETURN_GENERATOR", [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", - [GET_ITER] = "GET_ITER", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [FOR_ITER_GEN] = "FOR_ITER_GEN", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", - [LOAD_SUPER_ATTR_ATTR] = "LOAD_SUPER_ATTR_ATTR", - [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [RETURN_GENERATOR] = "RETURN_GENERATOR", - [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", - [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", - [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", + [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", [RETURN_VALUE] = "RETURN_VALUE", - [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", + [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", + [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [LOAD_LOCALS] = "LOAD_LOCALS", - [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", + [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", [DELETE_NAME] = "DELETE_NAME", @@ -346,9 +346,9 @@ static const char *const _PyOpcode_OpName[267] = { [IMPORT_NAME] = "IMPORT_NAME", [IMPORT_FROM] = "IMPORT_FROM", [JUMP_FORWARD] = "JUMP_FORWARD", - [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", + [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", + [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -367,7 +367,7 @@ static const char *const _PyOpcode_OpName[267] = { [POP_JUMP_IF_NONE] = "POP_JUMP_IF_NONE", [RAISE_VARARGS] = "RAISE_VARARGS", [GET_AWAITABLE] = "GET_AWAITABLE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [BUILD_SLICE] = "BUILD_SLICE", [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", [MAKE_CELL] = "MAKE_CELL", @@ -383,20 +383,20 @@ static const char *const _PyOpcode_OpName[267] = { [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", [MAP_ADD] = "MAP_ADD", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", + [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", [COPY_FREE_VARS] = "COPY_FREE_VARS", [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", + [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", + [CALL_NO_KW_LIST_APPEND] = "CALL_NO_KW_LIST_APPEND", + [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", [CONVERT_VALUE] = "CONVERT_VALUE", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", - [SEND_GEN] = "SEND_GEN", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS] = "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", + [CALL_NO_KW_METHOD_DESCRIPTOR_FAST] = "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", diff --git a/Include/opcode.h b/Include/opcode.h index 43a18065cf08c2..39bb70a8f2842f 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -158,65 +158,65 @@ extern "C" { #define LOAD_ZERO_SUPER_ATTR 265 #define STORE_FAST_MAYBE_NULL 266 #define MAX_PSEUDO_OPCODE 266 -#define BINARY_OP_ADD_FLOAT 6 +#define BINARY_OP_MULTIPLY_INT 6 #define BINARY_OP_ADD_INT 7 -#define BINARY_OP_ADD_UNICODE 8 -#define BINARY_OP_INPLACE_ADD_UNICODE 10 -#define BINARY_OP_MULTIPLY_FLOAT 13 -#define BINARY_OP_MULTIPLY_INT 14 -#define BINARY_OP_SUBTRACT_FLOAT 16 -#define BINARY_OP_SUBTRACT_INT 18 +#define BINARY_OP_SUBTRACT_INT 8 +#define BINARY_OP_MULTIPLY_FLOAT 10 +#define BINARY_OP_ADD_FLOAT 13 +#define BINARY_OP_SUBTRACT_FLOAT 14 +#define BINARY_OP_ADD_UNICODE 16 +#define BINARY_OP_INPLACE_ADD_UNICODE 18 #define BINARY_SUBSCR_DICT 19 #define BINARY_SUBSCR_GETITEM 20 #define BINARY_SUBSCR_LIST_INT 21 #define BINARY_SUBSCR_TUPLE_INT 22 -#define CALL_PY_EXACT_ARGS 23 -#define CALL_PY_WITH_DEFAULTS 28 -#define CALL_BOUND_METHOD_EXACT_ARGS 29 -#define CALL_BUILTIN_CLASS 34 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 38 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 39 -#define CALL_NO_KW_BUILTIN_FAST 42 -#define CALL_NO_KW_BUILTIN_O 43 -#define CALL_NO_KW_ISINSTANCE 44 -#define CALL_NO_KW_LEN 45 -#define CALL_NO_KW_LIST_APPEND 46 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 47 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 48 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 56 -#define CALL_NO_KW_STR_1 57 -#define CALL_NO_KW_TUPLE_1 58 -#define CALL_NO_KW_TYPE_1 59 -#define COMPARE_OP_FLOAT 62 -#define COMPARE_OP_INT 63 -#define COMPARE_OP_STR 64 -#define FOR_ITER_LIST 65 -#define FOR_ITER_TUPLE 66 -#define FOR_ITER_RANGE 67 -#define FOR_ITER_GEN 70 -#define LOAD_SUPER_ATTR_ATTR 72 -#define LOAD_SUPER_ATTR_METHOD 73 -#define LOAD_ATTR_CLASS 76 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 77 -#define LOAD_ATTR_INSTANCE_VALUE 78 -#define LOAD_ATTR_MODULE 79 -#define LOAD_ATTR_PROPERTY 80 -#define LOAD_ATTR_SLOT 81 -#define LOAD_ATTR_WITH_HINT 82 -#define LOAD_ATTR_METHOD_LAZY_DICT 84 -#define LOAD_ATTR_METHOD_NO_DICT 86 -#define LOAD_ATTR_METHOD_WITH_VALUES 88 -#define LOAD_GLOBAL_BUILTIN 111 -#define LOAD_GLOBAL_MODULE 112 -#define STORE_ATTR_INSTANCE_VALUE 113 -#define STORE_ATTR_SLOT 132 -#define STORE_ATTR_WITH_HINT 148 -#define STORE_SUBSCR_DICT 153 -#define STORE_SUBSCR_LIST_INT 154 -#define UNPACK_SEQUENCE_LIST 155 -#define UNPACK_SEQUENCE_TUPLE 159 -#define UNPACK_SEQUENCE_TWO_TUPLE 160 -#define SEND_GEN 161 +#define STORE_SUBSCR_DICT 23 +#define STORE_SUBSCR_LIST_INT 28 +#define SEND_GEN 29 +#define UNPACK_SEQUENCE_TWO_TUPLE 34 +#define UNPACK_SEQUENCE_TUPLE 38 +#define UNPACK_SEQUENCE_LIST 39 +#define STORE_ATTR_INSTANCE_VALUE 42 +#define STORE_ATTR_SLOT 43 +#define STORE_ATTR_WITH_HINT 44 +#define LOAD_GLOBAL_MODULE 45 +#define LOAD_GLOBAL_BUILTIN 46 +#define LOAD_SUPER_ATTR_ATTR 47 +#define LOAD_SUPER_ATTR_METHOD 48 +#define LOAD_ATTR_INSTANCE_VALUE 56 +#define LOAD_ATTR_MODULE 57 +#define LOAD_ATTR_WITH_HINT 58 +#define LOAD_ATTR_SLOT 59 +#define LOAD_ATTR_CLASS 62 +#define LOAD_ATTR_PROPERTY 63 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 64 +#define LOAD_ATTR_METHOD_WITH_VALUES 65 +#define LOAD_ATTR_METHOD_NO_DICT 66 +#define LOAD_ATTR_METHOD_LAZY_DICT 67 +#define COMPARE_OP_FLOAT 70 +#define COMPARE_OP_INT 72 +#define COMPARE_OP_STR 73 +#define FOR_ITER_LIST 76 +#define FOR_ITER_TUPLE 77 +#define FOR_ITER_RANGE 78 +#define FOR_ITER_GEN 79 +#define CALL_BOUND_METHOD_EXACT_ARGS 80 +#define CALL_PY_EXACT_ARGS 81 +#define CALL_PY_WITH_DEFAULTS 82 +#define CALL_NO_KW_TYPE_1 84 +#define CALL_NO_KW_STR_1 86 +#define CALL_NO_KW_TUPLE_1 88 +#define CALL_BUILTIN_CLASS 111 +#define CALL_NO_KW_BUILTIN_O 112 +#define CALL_NO_KW_BUILTIN_FAST 113 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 132 +#define CALL_NO_KW_LEN 148 +#define CALL_NO_KW_ISINSTANCE 153 +#define CALL_NO_KW_LIST_APPEND 154 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 155 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 159 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 160 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 161 #define NB_ADD 0 #define NB_AND 1 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py new file mode 100644 index 00000000000000..92ed4037d8ddb7 --- /dev/null +++ b/Lib/_opcode_metadata.py @@ -0,0 +1,94 @@ +# This file is generated by Tools/cases_generator/generate_cases.py +# from: +# Python/bytecodes.c +# Do not edit! + +_specializations = { + "BINARY_OP": [ + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_ADD_INT", + "BINARY_OP_SUBTRACT_INT", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_ADD_UNICODE", + ], + "BINARY_SUBSCR": [ + "BINARY_SUBSCR_DICT", + "BINARY_SUBSCR_GETITEM", + "BINARY_SUBSCR_LIST_INT", + "BINARY_SUBSCR_TUPLE_INT", + ], + "STORE_SUBSCR": [ + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", + ], + "SEND": [ + "SEND_GEN", + ], + "UNPACK_SEQUENCE": [ + "UNPACK_SEQUENCE_TWO_TUPLE", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_LIST", + ], + "STORE_ATTR": [ + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ], + "LOAD_GLOBAL": [ + "LOAD_GLOBAL_MODULE", + "LOAD_GLOBAL_BUILTIN", + ], + "LOAD_SUPER_ATTR": [ + "LOAD_SUPER_ATTR_ATTR", + "LOAD_SUPER_ATTR_METHOD", + ], + "LOAD_ATTR": [ + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_WITH_HINT", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_CLASS", + "LOAD_ATTR_PROPERTY", + "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + "LOAD_ATTR_METHOD_WITH_VALUES", + "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_LAZY_DICT", + ], + "COMPARE_OP": [ + "COMPARE_OP_FLOAT", + "COMPARE_OP_INT", + "COMPARE_OP_STR", + ], + "FOR_ITER": [ + "FOR_ITER_LIST", + "FOR_ITER_TUPLE", + "FOR_ITER_RANGE", + "FOR_ITER_GEN", + ], + "CALL": [ + "CALL_BOUND_METHOD_EXACT_ARGS", + "CALL_PY_EXACT_ARGS", + "CALL_PY_WITH_DEFAULTS", + "CALL_NO_KW_TYPE_1", + "CALL_NO_KW_STR_1", + "CALL_NO_KW_TUPLE_1", + "CALL_BUILTIN_CLASS", + "CALL_NO_KW_BUILTIN_O", + "CALL_NO_KW_BUILTIN_FAST", + "CALL_BUILTIN_FAST_WITH_KEYWORDS", + "CALL_NO_KW_LEN", + "CALL_NO_KW_ISINSTANCE", + "CALL_NO_KW_LIST_APPEND", + "CALL_NO_KW_METHOD_DESCRIPTOR_O", + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", + "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", + ], +} + +# An irregular case: +_specializations["BINARY_OP"].append("BINARY_OP_INPLACE_ADD_UNICODE") + +_specialized_instructions = [opcode for family in _specializations.values() for opcode in family] diff --git a/Lib/dis.py b/Lib/dis.py index f135a0bc9b4d98..1f07ba933749ed 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -13,10 +13,10 @@ _nb_ops, _intrinsic_1_descs, _intrinsic_2_descs, - _specializations, - _specialized_instructions, ) +from _opcode_metadata import _specializations, _specialized_instructions + __all__ = ["code_info", "dis", "disassemble", "distb", "disco", "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index cf68c448a6e4ec..6a3eabe1973ac8 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -450,6 +450,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3552 (Remove LOAD_FAST__LOAD_CONST and LOAD_CONST__LOAD_FAST) # Python 3.13a1 3553 (Add SET_FUNCTION_ATTRIBUTE) # Python 3.13a1 3554 (more efficient bytecodes for f-strings) +# Python 3.13a1 3555 (generate specialized opcodes metadata from bytecodes.c) # Python 3.14 will start with 3600 @@ -466,7 +467,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3554).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3555).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/opcode.py b/Lib/opcode.py index 71a8afad43fc6f..8bce4854c55c6d 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -348,96 +348,6 @@ def pseudo_op(name, op, real_ops): "INTRINSIC_SET_FUNCTION_TYPE_PARAMS", ] -_specializations = { - "BINARY_OP": [ - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_ADD_INT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_INPLACE_ADD_UNICODE", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_SUBTRACT_FLOAT", - "BINARY_OP_SUBTRACT_INT", - ], - "BINARY_SUBSCR": [ - "BINARY_SUBSCR_DICT", - "BINARY_SUBSCR_GETITEM", - "BINARY_SUBSCR_LIST_INT", - "BINARY_SUBSCR_TUPLE_INT", - ], - "CALL": [ - "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", - "CALL_BOUND_METHOD_EXACT_ARGS", - "CALL_BUILTIN_CLASS", - "CALL_BUILTIN_FAST_WITH_KEYWORDS", - "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - "CALL_NO_KW_BUILTIN_FAST", - "CALL_NO_KW_BUILTIN_O", - "CALL_NO_KW_ISINSTANCE", - "CALL_NO_KW_LEN", - "CALL_NO_KW_LIST_APPEND", - "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", - "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - "CALL_NO_KW_METHOD_DESCRIPTOR_O", - "CALL_NO_KW_STR_1", - "CALL_NO_KW_TUPLE_1", - "CALL_NO_KW_TYPE_1", - ], - "COMPARE_OP": [ - "COMPARE_OP_FLOAT", - "COMPARE_OP_INT", - "COMPARE_OP_STR", - ], - "FOR_ITER": [ - "FOR_ITER_LIST", - "FOR_ITER_TUPLE", - "FOR_ITER_RANGE", - "FOR_ITER_GEN", - ], - "LOAD_SUPER_ATTR": [ - "LOAD_SUPER_ATTR_ATTR", - "LOAD_SUPER_ATTR_METHOD", - ], - "LOAD_ATTR": [ - # These potentially push [NULL, bound method] onto the stack. - "LOAD_ATTR_CLASS", - "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_MODULE", - "LOAD_ATTR_PROPERTY", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_WITH_HINT", - # These will always push [unbound method, self] onto the stack. - "LOAD_ATTR_METHOD_LAZY_DICT", - "LOAD_ATTR_METHOD_NO_DICT", - "LOAD_ATTR_METHOD_WITH_VALUES", - ], - "LOAD_GLOBAL": [ - "LOAD_GLOBAL_BUILTIN", - "LOAD_GLOBAL_MODULE", - ], - "STORE_ATTR": [ - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", - ], - "STORE_SUBSCR": [ - "STORE_SUBSCR_DICT", - "STORE_SUBSCR_LIST_INT", - ], - "UNPACK_SEQUENCE": [ - "UNPACK_SEQUENCE_LIST", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_TWO_TUPLE", - ], - "SEND": [ - "SEND_GEN", - ], -} -_specialized_instructions = [ - opcode for family in _specializations.values() for opcode in family -] _cache_format = { "LOAD_GLOBAL": { diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 7dd6e3f0d62439..ff7e81970e584d 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -12,6 +12,7 @@ from test.support.bytecode_helper import BytecodeTestCase import opcode +import _opcode_metadata def get_tb(): @@ -1916,12 +1917,12 @@ def test_baseopname_and_baseopcode(self): self.assertEqual(code, baseopcode) # Specialized instructions - for name in opcode._specialized_instructions: + for name in _opcode_metadata._specialized_instructions: instruction = Instruction(opname=name, opcode=dis._all_opmap[name], arg=None, argval=None, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None) baseopname = instruction.baseopname baseopcode = instruction.baseopcode - self.assertIn(name, opcode._specializations[baseopname]) + self.assertIn(name, _opcode_metadata._specializations[baseopname]) self.assertEqual(opcode.opmap[baseopname], baseopcode) def test_jump_target(self): diff --git a/Makefile.pre.in b/Makefile.pre.in index a9c53bae4bf63c..a8cc00a5b3f554 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1427,7 +1427,8 @@ regen-opcode: $(srcdir)/Lib/opcode.py \ $(srcdir)/Include/opcode.h.new \ $(srcdir)/Include/internal/pycore_opcode.h.new \ - $(srcdir)/Include/internal/pycore_intrinsics.h.new + $(srcdir)/Include/internal/pycore_intrinsics.h.new \ + $(srcdir)/Lib/_opcode_metadata.py $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_intrinsics.h $(srcdir)/Include/internal/pycore_intrinsics.h.new diff --git a/Python/makeopcodetargets.py b/Python/makeopcodetargets.py index 2b402ae0b6a031..5843079b729936 100755 --- a/Python/makeopcodetargets.py +++ b/Python/makeopcodetargets.py @@ -25,12 +25,13 @@ def write_contents(f): """Write C code contents to the target file object. """ opcode = find_module('opcode') + _opcode_metadata = find_module('_opcode_metadata') targets = ['_unknown_opcode'] * 256 for opname, op in opcode.opmap.items(): if not opcode.is_pseudo(op): targets[op] = "TARGET_%s" % opname next_op = 1 - for opname in opcode._specialized_instructions: + for opname in _opcode_metadata._specialized_instructions: while targets[next_op] != '_unknown_opcode': next_op += 1 targets[next_op] = "TARGET_%s" % opname diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 451d8bbd17fa63..4de4bf0608f35e 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -5,49 +5,49 @@ static void *opcode_targets[256] = { &&TARGET_INTERPRETER_EXIT, &&TARGET_END_FOR, &&TARGET_END_SEND, - &&TARGET_BINARY_OP_ADD_FLOAT, + &&TARGET_BINARY_OP_MULTIPLY_INT, &&TARGET_BINARY_OP_ADD_INT, - &&TARGET_BINARY_OP_ADD_UNICODE, + &&TARGET_BINARY_OP_SUBTRACT_INT, &&TARGET_NOP, - &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, + &&TARGET_BINARY_OP_MULTIPLY_FLOAT, &&TARGET_UNARY_NEGATIVE, &&TARGET_UNARY_NOT, - &&TARGET_BINARY_OP_MULTIPLY_FLOAT, - &&TARGET_BINARY_OP_MULTIPLY_INT, - &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_ADD_FLOAT, &&TARGET_BINARY_OP_SUBTRACT_FLOAT, + &&TARGET_UNARY_INVERT, + &&TARGET_BINARY_OP_ADD_UNICODE, &&TARGET_RESERVED, - &&TARGET_BINARY_OP_SUBTRACT_INT, + &&TARGET_BINARY_OP_INPLACE_ADD_UNICODE, &&TARGET_BINARY_SUBSCR_DICT, &&TARGET_BINARY_SUBSCR_GETITEM, &&TARGET_BINARY_SUBSCR_LIST_INT, &&TARGET_BINARY_SUBSCR_TUPLE_INT, - &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_STORE_SUBSCR_DICT, &&TARGET_MAKE_FUNCTION, &&TARGET_BINARY_SUBSCR, &&TARGET_BINARY_SLICE, &&TARGET_STORE_SLICE, - &&TARGET_CALL_PY_WITH_DEFAULTS, - &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_STORE_SUBSCR_LIST_INT, + &&TARGET_SEND_GEN, &&TARGET_GET_LEN, &&TARGET_MATCH_MAPPING, &&TARGET_MATCH_SEQUENCE, &&TARGET_MATCH_KEYS, - &&TARGET_CALL_BUILTIN_CLASS, + &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, &&TARGET_PUSH_EXC_INFO, &&TARGET_CHECK_EXC_MATCH, &&TARGET_CHECK_EG_MATCH, - &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, - &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_UNPACK_SEQUENCE_TUPLE, + &&TARGET_UNPACK_SEQUENCE_LIST, &&TARGET_FORMAT_SIMPLE, &&TARGET_FORMAT_WITH_SPEC, - &&TARGET_CALL_NO_KW_BUILTIN_FAST, - &&TARGET_CALL_NO_KW_BUILTIN_O, - &&TARGET_CALL_NO_KW_ISINSTANCE, - &&TARGET_CALL_NO_KW_LEN, - &&TARGET_CALL_NO_KW_LIST_APPEND, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_STORE_ATTR_SLOT, + &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_LOAD_GLOBAL_MODULE, + &&TARGET_LOAD_GLOBAL_BUILTIN, + &&TARGET_LOAD_SUPER_ATTR_ATTR, + &&TARGET_LOAD_SUPER_ATTR_METHOD, &&TARGET_WITH_EXCEPT_START, &&TARGET_GET_AITER, &&TARGET_GET_ANEXT, @@ -55,39 +55,39 @@ static void *opcode_targets[256] = { &&TARGET_BEFORE_WITH, &&TARGET_END_ASYNC_FOR, &&TARGET_CLEANUP_THROW, - &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, - &&TARGET_CALL_NO_KW_STR_1, - &&TARGET_CALL_NO_KW_TUPLE_1, - &&TARGET_CALL_NO_KW_TYPE_1, + &&TARGET_LOAD_ATTR_INSTANCE_VALUE, + &&TARGET_LOAD_ATTR_MODULE, + &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_LOAD_ATTR_SLOT, &&TARGET_STORE_SUBSCR, &&TARGET_DELETE_SUBSCR, + &&TARGET_LOAD_ATTR_CLASS, + &&TARGET_LOAD_ATTR_PROPERTY, + &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, + &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_GET_ITER, + &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_COMPARE_OP_FLOAT, + &&TARGET_LOAD_BUILD_CLASS, &&TARGET_COMPARE_OP_INT, &&TARGET_COMPARE_OP_STR, + &&TARGET_LOAD_ASSERTION_ERROR, + &&TARGET_RETURN_GENERATOR, &&TARGET_FOR_ITER_LIST, &&TARGET_FOR_ITER_TUPLE, &&TARGET_FOR_ITER_RANGE, - &&TARGET_GET_ITER, - &&TARGET_GET_YIELD_FROM_ITER, &&TARGET_FOR_ITER_GEN, - &&TARGET_LOAD_BUILD_CLASS, - &&TARGET_LOAD_SUPER_ATTR_ATTR, - &&TARGET_LOAD_SUPER_ATTR_METHOD, - &&TARGET_LOAD_ASSERTION_ERROR, - &&TARGET_RETURN_GENERATOR, - &&TARGET_LOAD_ATTR_CLASS, - &&TARGET_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN, - &&TARGET_LOAD_ATTR_INSTANCE_VALUE, - &&TARGET_LOAD_ATTR_MODULE, - &&TARGET_LOAD_ATTR_PROPERTY, - &&TARGET_LOAD_ATTR_SLOT, - &&TARGET_LOAD_ATTR_WITH_HINT, + &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_PY_EXACT_ARGS, + &&TARGET_CALL_PY_WITH_DEFAULTS, &&TARGET_RETURN_VALUE, - &&TARGET_LOAD_ATTR_METHOD_LAZY_DICT, + &&TARGET_CALL_NO_KW_TYPE_1, &&TARGET_SETUP_ANNOTATIONS, - &&TARGET_LOAD_ATTR_METHOD_NO_DICT, + &&TARGET_CALL_NO_KW_STR_1, &&TARGET_LOAD_LOCALS, - &&TARGET_LOAD_ATTR_METHOD_WITH_VALUES, + &&TARGET_CALL_NO_KW_TUPLE_1, &&TARGET_POP_EXCEPT, &&TARGET_STORE_NAME, &&TARGET_DELETE_NAME, @@ -110,9 +110,9 @@ static void *opcode_targets[256] = { &&TARGET_IMPORT_NAME, &&TARGET_IMPORT_FROM, &&TARGET_JUMP_FORWARD, - &&TARGET_LOAD_GLOBAL_BUILTIN, - &&TARGET_LOAD_GLOBAL_MODULE, - &&TARGET_STORE_ATTR_INSTANCE_VALUE, + &&TARGET_CALL_BUILTIN_CLASS, + &&TARGET_CALL_NO_KW_BUILTIN_O, + &&TARGET_CALL_NO_KW_BUILTIN_FAST, &&TARGET_POP_JUMP_IF_FALSE, &&TARGET_POP_JUMP_IF_TRUE, &&TARGET_LOAD_GLOBAL, @@ -131,7 +131,7 @@ static void *opcode_targets[256] = { &&TARGET_POP_JUMP_IF_NONE, &&TARGET_RAISE_VARARGS, &&TARGET_GET_AWAITABLE, - &&TARGET_STORE_ATTR_SLOT, + &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, &&TARGET_BUILD_SLICE, &&TARGET_JUMP_BACKWARD_NO_INTERRUPT, &&TARGET_MAKE_CELL, @@ -147,20 +147,20 @@ static void *opcode_targets[256] = { &&TARGET_LIST_APPEND, &&TARGET_SET_ADD, &&TARGET_MAP_ADD, - &&TARGET_STORE_ATTR_WITH_HINT, + &&TARGET_CALL_NO_KW_LEN, &&TARGET_COPY_FREE_VARS, &&TARGET_YIELD_VALUE, &&TARGET_RESUME, &&TARGET_MATCH_CLASS, - &&TARGET_STORE_SUBSCR_DICT, - &&TARGET_STORE_SUBSCR_LIST_INT, - &&TARGET_UNPACK_SEQUENCE_LIST, + &&TARGET_CALL_NO_KW_ISINSTANCE, + &&TARGET_CALL_NO_KW_LIST_APPEND, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_O, &&TARGET_BUILD_CONST_KEY_MAP, &&TARGET_BUILD_STRING, &&TARGET_CONVERT_VALUE, - &&TARGET_UNPACK_SEQUENCE_TUPLE, - &&TARGET_UNPACK_SEQUENCE_TWO_TUPLE, - &&TARGET_SEND_GEN, + &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS, + &&TARGET_CALL_NO_KW_METHOD_DESCRIPTOR_FAST, &&TARGET_LIST_EXTEND, &&TARGET_SET_UPDATE, &&TARGET_DICT_MERGE, diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 925b8b3230fce3..13b1764f0886d1 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -47,6 +47,7 @@ static const char* _Py_stdlib_module_names[] = { "_multibytecodec", "_multiprocessing", "_opcode", +"_opcode_metadata", "_operator", "_osx_support", "_overlapped", diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index ec9fdae56128c0..bb86c4c3d0cc01 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -66,18 +66,21 @@ UINT32_MASK = (1<<32)-1 +def get_python_module_dict(filename): + mod = {} + with tokenize.open(filename) as fp: + code = fp.read() + exec(code, mod) + return mod def main(opcode_py, outfile='Include/opcode.h', internaloutfile='Include/internal/pycore_opcode.h', - intrinsicoutfile='Include/internal/pycore_intrinsics.h'): - opcode = {} - if hasattr(tokenize, 'open'): - fp = tokenize.open(opcode_py) # Python 3.2+ - else: - fp = open(opcode_py) # Python 2.7 - with fp: - code = fp.read() - exec(code, opcode) + intrinsicoutfile='Include/internal/pycore_intrinsics.h', + _opcode_metadata_py='Lib/_opcode_metadata.py'): + + _opcode_metadata = get_python_module_dict(_opcode_metadata_py) + + opcode = get_python_module_dict(opcode_py) opmap = opcode['opmap'] opname = opcode['opname'] hasarg = opcode['hasarg'] @@ -87,6 +90,7 @@ def main(opcode_py, outfile='Include/opcode.h', is_pseudo = opcode['is_pseudo'] _pseudo_ops = opcode['_pseudo_ops'] + ENABLE_SPECIALIZATION = opcode["ENABLE_SPECIALIZATION"] MIN_PSEUDO_OPCODE = opcode["MIN_PSEUDO_OPCODE"] MAX_PSEUDO_OPCODE = opcode["MAX_PSEUDO_OPCODE"] @@ -101,7 +105,7 @@ def main(opcode_py, outfile='Include/opcode.h', specialized_opmap = {} opname_including_specialized = opname.copy() - for name in opcode['_specialized_instructions']: + for name in _opcode_metadata['_specialized_instructions']: while used[next_op]: next_op += 1 specialized_opmap[name] = next_op @@ -145,7 +149,7 @@ def main(opcode_py, outfile='Include/opcode.h', for basic, op in opmap.items(): if not is_pseudo(op): deoptcodes[basic] = basic - for basic, family in opcode["_specializations"].items(): + for basic, family in _opcode_metadata["_specializations"].items(): for specialized in family: deoptcodes[specialized] = basic iobj.write("\nconst uint8_t _PyOpcode_Deopt[256] = {\n") @@ -203,4 +207,4 @@ def main(opcode_py, outfile='Include/opcode.h', if __name__ == '__main__': - main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) + main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4], sys.argv[5]) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index 723996bd8f7696..a28a0cc9377f6a 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -26,6 +26,9 @@ DEFAULT_METADATA_OUTPUT = os.path.relpath( os.path.join(ROOT, "Python/opcode_metadata.h") ) +DEFAULT_PYMETADATA_OUTPUT = os.path.relpath( + os.path.join(ROOT, "LIB/_opcode_metadata.py") +) BEGIN_MARKER = "// BEGIN BYTECODES //" END_MARKER = "// END BYTECODES //" RE_PREDICTED = ( @@ -47,7 +50,10 @@ "-o", "--output", type=str, help="Generated code", default=DEFAULT_OUTPUT ) arg_parser.add_argument( - "-m", "--metadata", type=str, help="Generated metadata", default=DEFAULT_METADATA_OUTPUT + "-m", "--metadata", type=str, help="Generated C metadata", default=DEFAULT_METADATA_OUTPUT +) +arg_parser.add_argument( + "-p", "--pymetadata", type=str, help="Generated Python metadata", default=DEFAULT_PYMETADATA_OUTPUT ) arg_parser.add_argument( "-l", "--emit-line-directives", help="Emit #line directives", action="store_true" @@ -121,11 +127,13 @@ class Formatter: nominal_filename: str def __init__( - self, stream: typing.TextIO, indent: int, emit_line_directives: bool = False + self, stream: typing.TextIO, indent: int, + emit_line_directives: bool = False, comment: str = "//", ) -> None: self.stream = stream self.prefix = " " * indent self.emit_line_directives = emit_line_directives + self.comment = comment self.lineno = 1 filename = os.path.relpath(self.stream.name, ROOT) # Make filename more user-friendly and less platform-specific @@ -519,14 +527,17 @@ class Analyzer: input_filenames: list[str] output_filename: str metadata_filename: str + pymetadata_filename: str errors: int = 0 emit_line_directives: bool = False - def __init__(self, input_filenames: list[str], output_filename: str, metadata_filename: str): + def __init__(self, input_filenames: list[str], output_filename: str, + metadata_filename: str, pymetadata_filename: str): """Read the input file.""" self.input_filenames = input_filenames self.output_filename = output_filename self.metadata_filename = metadata_filename + self.pymetadata_filename = pymetadata_filename def error(self, msg: str, node: parser.Node) -> None: lineno = 0 @@ -1005,11 +1016,16 @@ def write_function( self.out.emit("") def from_source_files(self) -> str: - paths = "\n// ".join( + paths = f"\n{self.out.comment} ".join( os.path.relpath(filename, ROOT).replace(os.path.sep, posixpath.sep) for filename in self.input_filenames ) - return f"// from:\n// {paths}\n" + return f"{self.out.comment} from:\n{self.out.comment} {paths}\n" + + def write_provenance_header(self): + self.out.write_raw(f"{self.out.comment} This file is generated by {THIS}\n") + self.out.write_raw(self.from_source_files()) + self.out.write_raw(f"{self.out.comment} Do not edit!\n") def write_metadata(self) -> None: """Write instruction metadata to output file.""" @@ -1043,10 +1059,7 @@ def write_metadata(self) -> None: # Create formatter self.out = Formatter(f, 0) - # Write provenance header - self.out.write_raw(f"// This file is generated by {THIS}\n") - self.out.write_raw(self.from_source_files()) - self.out.write_raw(f"// Do not edit!\n") + self.write_provenance_header() self.write_pseudo_instrs() @@ -1100,6 +1113,39 @@ def write_metadata(self) -> None: self.out.emit("};") self.out.emit("#endif") + with open(self.pymetadata_filename, "w") as f: + # Create formatter + self.out = Formatter(f, 0, comment = "#") + + self.write_provenance_header() + + self.out.emit("") + self.out.emit("_specializations = {") + for name, family in self.families.items(): + assert len(family.members) > 1 + with self.out.indent(): + self.out.emit(f"\"{family.members[0]}\": [") + with self.out.indent(): + for m in family.members[1:]: + self.out.emit(f"\"{m}\",") + self.out.emit(f"],") + self.out.emit("}") + + # Handle special case + self.out.emit("") + self.out.emit("# An irregular case:") + self.out.emit( + "_specializations[\"BINARY_OP\"].append(" + "\"BINARY_OP_INPLACE_ADD_UNICODE\")") + + # Make list of specialized instructions + self.out.emit("") + self.out.emit( + "_specialized_instructions = [" + "opcode for family in _specializations.values() for opcode in family" + "]") + + def write_pseudo_instrs(self) -> None: """Write the IS_PSEUDO_INSTR macro""" self.out.emit("\n\n#define IS_PSEUDO_INSTR(OP) \\") @@ -1134,9 +1180,9 @@ def write_instructions(self) -> None: self.out = Formatter(f, 8, self.emit_line_directives) # Write provenance header - self.out.write_raw(f"// This file is generated by {THIS}\n") + self.out.write_raw(f"{self.out.comment} This file is generated by {THIS}\n") self.out.write_raw(self.from_source_files()) - self.out.write_raw(f"// Do not edit!\n") + self.out.write_raw(f"{self.out.comment} Do not edit!\n") # Write and count instructions of all kinds n_instrs = 0 @@ -1168,13 +1214,13 @@ def write_overridden_instr_place_holder(self, place_holder: OverriddenInstructionPlaceHolder) -> None: self.out.emit("") self.out.emit( - f"// TARGET({place_holder.name}) overridden by later definition") + f"{self.out.comment} TARGET({place_holder.name}) overridden by later definition") def write_instr(self, instr: Instruction) -> None: name = instr.name self.out.emit("") if instr.inst.override: - self.out.emit("// Override") + self.out.emit("{self.out.comment} Override") with self.out.block(f"TARGET({name})"): if instr.predicted: self.out.emit(f"PREDICTED({name});") @@ -1315,7 +1361,10 @@ def main(): args = arg_parser.parse_args() # Prints message and sys.exit(2) on error if len(args.input) == 0: args.input.append(DEFAULT_INPUT) - a = Analyzer(args.input, args.output, args.metadata) # Raises OSError if input unreadable + + # Raises OSError if input unreadable + a = Analyzer(args.input, args.output, args.metadata, args.pymetadata) + if args.emit_line_directives: a.emit_line_directives = True a.parse() # Raises SyntaxError on failure From 66314d9ab843e67fb0330eb55b3d7f12d1412839 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 19 Jun 2023 15:01:27 +0100 Subject: [PATCH 2/9] update PCbuild/regen.targets --- PCbuild/regen.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 15f3d1375a10a2..c6364e84632373 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -59,7 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - From 19aa0c79b6bb86d4d5848c9759de8685c53d19bc Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 19 Jun 2023 15:06:18 +0100 Subject: [PATCH 3/9] update a couple of tests --- Lib/test/test__opcode.py | 3 ++- Lib/test/test_embed.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index 3e084928844d2f..cdeda21aa2aef3 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -2,6 +2,7 @@ from test.support.import_helper import import_module import unittest import opcode +import _opcode_metadata _opcode = import_module("_opcode") from _opcode import stack_effect @@ -66,7 +67,7 @@ def test_specialization_stats(self): stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"] specialized_opcodes = [ op.lower() - for op in opcode._specializations + for op in _opcode_metadata._specializations if opcode._inline_cache_entries[opcode.opmap[op]] ] self.assertIn('load_attr', specialized_opcodes) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 582392ecddcb91..1582f5caf73dfe 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -355,13 +355,14 @@ def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self): import dis import importlib._bootstrap import opcode + import _opcode_metadata import test.test_dis def is_specialized(f): for instruction in dis.get_instructions(f, adaptive=True): opname = instruction.opname if ( - opname in opcode._specialized_instructions + opname in _opcode_metadata._specialized_instructions # Exclude superinstructions: and "__" not in opname ): From b7ae5cd924c8a5051eef77c8468bceb94a408545 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 19 Jun 2023 16:52:37 +0100 Subject: [PATCH 4/9] include stuff from _opcode_metadata in opcode --- Lib/dis.py | 4 ++-- Lib/opcode.py | 2 ++ Lib/test/test__opcode.py | 3 +-- Lib/test/test_embed.py | 3 +-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/dis.py b/Lib/dis.py index 1f07ba933749ed..f135a0bc9b4d98 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -13,10 +13,10 @@ _nb_ops, _intrinsic_1_descs, _intrinsic_2_descs, + _specializations, + _specialized_instructions, ) -from _opcode_metadata import _specializations, _specialized_instructions - __all__ = ["code_info", "dis", "disassemble", "distb", "disco", "findlinestarts", "findlabels", "show_code", "get_instructions", "Instruction", "Bytecode"] + _opcodes_all diff --git a/Lib/opcode.py b/Lib/opcode.py index 8bce4854c55c6d..819e35a41e589b 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -21,6 +21,8 @@ except ImportError: pass +from _opcode_metadata import _specializations, _specialized_instructions + cmp_op = ('<', '<=', '==', '!=', '>', '>=') hasarg = [] diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py index cdeda21aa2aef3..3e084928844d2f 100644 --- a/Lib/test/test__opcode.py +++ b/Lib/test/test__opcode.py @@ -2,7 +2,6 @@ from test.support.import_helper import import_module import unittest import opcode -import _opcode_metadata _opcode = import_module("_opcode") from _opcode import stack_effect @@ -67,7 +66,7 @@ def test_specialization_stats(self): stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"] specialized_opcodes = [ op.lower() - for op in _opcode_metadata._specializations + for op in opcode._specializations if opcode._inline_cache_entries[opcode.opmap[op]] ] self.assertIn('load_attr', specialized_opcodes) diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1582f5caf73dfe..582392ecddcb91 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -355,14 +355,13 @@ def test_specialized_static_code_gets_unspecialized_at_Py_FINALIZE(self): import dis import importlib._bootstrap import opcode - import _opcode_metadata import test.test_dis def is_specialized(f): for instruction in dis.get_instructions(f, adaptive=True): opname = instruction.opname if ( - opname in _opcode_metadata._specialized_instructions + opname in opcode._specialized_instructions # Exclude superinstructions: and "__" not in opname ): From d8fac92536a09b43fc08f241de355e44be38f4d5 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 19 Jun 2023 17:36:56 +0100 Subject: [PATCH 5/9] update import --- Lib/test/test_dis.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index ff7e81970e584d..7dd6e3f0d62439 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -12,7 +12,6 @@ from test.support.bytecode_helper import BytecodeTestCase import opcode -import _opcode_metadata def get_tb(): @@ -1917,12 +1916,12 @@ def test_baseopname_and_baseopcode(self): self.assertEqual(code, baseopcode) # Specialized instructions - for name in _opcode_metadata._specialized_instructions: + for name in opcode._specialized_instructions: instruction = Instruction(opname=name, opcode=dis._all_opmap[name], arg=None, argval=None, argrepr='', offset=0, start_offset=0, starts_line=1, is_jump_target=False, positions=None) baseopname = instruction.baseopname baseopcode = instruction.baseopcode - self.assertIn(name, _opcode_metadata._specializations[baseopname]) + self.assertIn(name, opcode._specializations[baseopname]) self.assertEqual(opcode.opmap[baseopname], baseopcode) def test_jump_target(self): From 854ddf3d19810d8a54b937f6aa5b2b077f344615 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 19 Jun 2023 18:13:37 +0100 Subject: [PATCH 6/9] tolerate _opcode_metadata not being available in early stages of the build --- Lib/opcode.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Lib/opcode.py b/Lib/opcode.py index 819e35a41e589b..ed01d2cdad9ebf 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -21,7 +21,11 @@ except ImportError: pass -from _opcode_metadata import _specializations, _specialized_instructions +# _opcode_metadata may not be ready during early stages of the build +try: + from _opcode_metadata import _specializations, _specialized_instructions +except ModuleNotFoundError: + pass cmp_op = ('<', '<=', '==', '!=', '>', '>=') From 7ed6716ef8f22aa36304a990bd8d312bc563eb69 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 19 Jun 2023 18:50:13 +0100 Subject: [PATCH 7/9] fix case --- Tools/cases_generator/generate_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index a28a0cc9377f6a..e4ccd38bac4b49 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -27,7 +27,7 @@ os.path.join(ROOT, "Python/opcode_metadata.h") ) DEFAULT_PYMETADATA_OUTPUT = os.path.relpath( - os.path.join(ROOT, "LIB/_opcode_metadata.py") + os.path.join(ROOT, "Lib/_opcode_metadata.py") ) BEGIN_MARKER = "// BEGIN BYTECODES //" END_MARKER = "// END BYTECODES //" From 4556c5efda4cc8f82d4eb22d2e9a6b9f0092732c Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 19 Jun 2023 21:50:38 +0100 Subject: [PATCH 8/9] reorder args --- Makefile.pre.in | 4 ++-- PCbuild/regen.targets | 2 +- Tools/build/generate_opcode_h.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index a8cc00a5b3f554..3a404b0d16403f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1425,10 +1425,10 @@ regen-opcode: # using Tools/build/generate_opcode_h.py $(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \ $(srcdir)/Lib/opcode.py \ + $(srcdir)/Lib/_opcode_metadata.py \ $(srcdir)/Include/opcode.h.new \ $(srcdir)/Include/internal/pycore_opcode.h.new \ - $(srcdir)/Include/internal/pycore_intrinsics.h.new \ - $(srcdir)/Lib/_opcode_metadata.py + $(srcdir)/Include/internal/pycore_intrinsics.h.new $(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_opcode.h $(srcdir)/Include/internal/pycore_opcode.h.new $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_intrinsics.h $(srcdir)/Include/internal/pycore_intrinsics.h.new diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index c6364e84632373..e9e16a15f94cce 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -59,7 +59,7 @@ Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)" DependsOnTargets="FindPythonForBuild"> - diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index bb86c4c3d0cc01..34564337513105 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -74,9 +74,9 @@ def get_python_module_dict(filename): return mod def main(opcode_py, outfile='Include/opcode.h', + _opcode_metadata_py='Lib/_opcode_metadata.py', internaloutfile='Include/internal/pycore_opcode.h', - intrinsicoutfile='Include/internal/pycore_intrinsics.h', - _opcode_metadata_py='Lib/_opcode_metadata.py'): + intrinsicoutfile='Include/internal/pycore_intrinsics.h'): _opcode_metadata = get_python_module_dict(_opcode_metadata_py) From d9d3b80d3dfc64e4b48dd84ea386e1d5764c2b39 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 19 Jun 2023 23:02:30 +0100 Subject: [PATCH 9/9] typo --- Tools/build/generate_opcode_h.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/build/generate_opcode_h.py b/Tools/build/generate_opcode_h.py index 34564337513105..4711fbbd1eb8c3 100644 --- a/Tools/build/generate_opcode_h.py +++ b/Tools/build/generate_opcode_h.py @@ -73,8 +73,9 @@ def get_python_module_dict(filename): exec(code, mod) return mod -def main(opcode_py, outfile='Include/opcode.h', +def main(opcode_py, _opcode_metadata_py='Lib/_opcode_metadata.py', + outfile='Include/opcode.h', internaloutfile='Include/internal/pycore_opcode.h', intrinsicoutfile='Include/internal/pycore_intrinsics.h'):