Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

GH-98686: Get rid of "adaptive" and "quick" instructions #99182

Merged
merged 25 commits into from
Nov 9, 2022
Merged

GH-98686: Get rid of "adaptive" and "quick" instructions #99182

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
2dc5e71
Combine all unquickened and adaptive instructions
brandtbucher Oct 20, 2022
15c806f
Merge EXTENDED_ARG and EXTENDED_ARG_QUICK
brandtbucher Oct 20, 2022
7c164d2
Add back BINARY_OP_GENERIC and COMPARE_OP_GENERIC
brandtbucher Oct 20, 2022
2eda520
Merge the miss and backoff counters
brandtbucher Oct 20, 2022
e36c5d7
make regen-cases
brandtbucher Oct 20, 2022
3caa5d4
Try inlining the miss label everywhere
brandtbucher Oct 20, 2022
db558f1
Update adaptive.md
brandtbucher Oct 21, 2022
b8796e6
Remove branching from EXTENDED_ARG
brandtbucher Oct 21, 2022
a7a451b
fixup
brandtbucher Oct 21, 2022
6fdf5ad
Remove tracing branches in adaptive instructions
brandtbucher Oct 21, 2022
e69e254
Remove error checking from many specializations
brandtbucher Oct 21, 2022
513aaab
Fix macro
brandtbucher Oct 21, 2022
a5c6cab
Catch up with main
brandtbucher Oct 21, 2022
1cd6d66
Make sure stats overhead is disabled be default
brandtbucher Oct 21, 2022
ea175fc
Use a single direct jump for misses
brandtbucher Nov 5, 2022
4dbff4d
Catch up with main
brandtbucher Nov 5, 2022
553ebab
Revert some unrelated changes
brandtbucher Nov 5, 2022
dc545bd
Clean up the diff
brandtbucher Nov 5, 2022
3639b66
Clarify the reasoning behind each counter value
brandtbucher Nov 5, 2022
84bc481
Merge EXTENDED_ARG and EXTENDED_ARG_QUICK
brandtbucher Nov 5, 2022
6e60694
Catch up with main
brandtbucher Nov 5, 2022
f885e6c
Fix stats
brandtbucher Nov 5, 2022
f33d882
fixup
brandtbucher Nov 5, 2022
cf98d4a
blurb add
brandtbucher Nov 6, 2022
58698db
Catch up with main
brandtbucher Oct 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge the miss and backoff counters
  • Loading branch information
@brandtbucher
brandtbucher committed Oct 20, 2022
commit 2eda5205b0c7784736090adc6dfcc0c8e761523f
18 changes: 14 additions & 4 deletions Include/internal/pycore_code.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -375,8 +375,12 @@ write_location_entry_start(uint8_t *ptr, int code, int length)

/* With a 16-bit counter, we have 12 bits for the counter value, and 4 bits for the backoff */
#define ADAPTIVE_BACKOFF_BITS 4
/* The initial counter value is 1 == 2**ADAPTIVE_BACKOFF_START - 1 */
#define ADAPTIVE_BACKOFF_START 1

#define ADAPTIVE_BACKOFF_START_VALUE 1
#define ADAPTIVE_BACKOFF_START_BACKOFF 1 // 2**0 <= 1 < 2**1

#define ADAPTIVE_BACKOFF_RESTART_VALUE 52
#define ADAPTIVE_BACKOFF_RESTART_BACKOFF 6 // 2**5 <= 53 < 2**6

#define MAX_BACKOFF_VALUE (16 - ADAPTIVE_BACKOFF_BITS)

Expand All @@ -389,8 +393,14 @@ adaptive_counter_bits(int value, int backoff) {

static inline uint16_t
adaptive_counter_start(void) {
unsigned int value = (1 << ADAPTIVE_BACKOFF_START) - 1;
return adaptive_counter_bits(value, ADAPTIVE_BACKOFF_START);
return adaptive_counter_bits(ADAPTIVE_BACKOFF_START_VALUE,
ADAPTIVE_BACKOFF_START_BACKOFF);
}

static inline uint16_t
adaptive_counter_restart(void) {
return adaptive_counter_bits(ADAPTIVE_BACKOFF_RESTART_VALUE,
ADAPTIVE_BACKOFF_RESTART_BACKOFF);
}

static inline uint16_t
Expand Down
220 changes: 100 additions & 120 deletions Python/bytecodes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -471,19 +471,17 @@ dummy_func(

// stack effect: (__0 -- )
inst(BINARY_SUBSCR) {
_PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
if (cframe.use_tracing || opcode != BINARY_SUBSCR) {
}
else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *sub = TOP();
PyObject *container = SECOND();
next_instr--;
if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) {
goto error;
if (!cframe.use_tracing) {
_PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *sub = TOP();
PyObject *container = SECOND();
next_instr--;
if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(BINARY_SUBSCR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
}
Expand Down Expand Up @@ -627,19 +625,17 @@ dummy_func(

// stack effect: (__0, __1, __2 -- )
inst(STORE_SUBSCR) {
_PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
if (cframe.use_tracing || opcode != STORE_SUBSCR) {
}
else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *sub = TOP();
PyObject *container = SECOND();
next_instr--;
if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) {
goto error;
if (!cframe.use_tracing) {
_PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *sub = TOP();
PyObject *container = SECOND();
next_instr--;
if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(STORE_SUBSCR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
}
Expand Down Expand Up @@ -1196,16 +1192,14 @@ dummy_func(

// stack effect: (__0 -- __array[oparg])
inst(UNPACK_SEQUENCE) {
_PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;
if (cframe.use_tracing || opcode != UNPACK_SEQUENCE) {
}
else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *seq = TOP();
next_instr--;
_Py_Specialize_UnpackSequence(seq, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
else {
if (!cframe.use_tracing) {
_PyUnpackSequenceCache *cache = (_PyUnpackSequenceCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *seq = TOP();
next_instr--;
_Py_Specialize_UnpackSequence(seq, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
STAT_INC(UNPACK_SEQUENCE, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
}
Expand Down Expand Up @@ -1377,18 +1371,16 @@ dummy_func(

// error: LOAD_GLOBAL has irregular stack effect
inst(LOAD_GLOBAL) {
_PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
if (cframe.use_tracing || opcode != LOAD_GLOBAL) {
}
else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *name = GETITEM(names, oparg>>1);
next_instr--;
if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) {
goto error;
if (!cframe.use_tracing) {
_PyLoadGlobalCache *cache = (_PyLoadGlobalCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *name = GETITEM(names, oparg>>1);
next_instr--;
if (_Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name) < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(LOAD_GLOBAL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
}
Expand Down Expand Up @@ -1836,19 +1828,17 @@ dummy_func(

// error: LOAD_ATTR has irregular stack effect
inst(LOAD_ATTR) {
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
if (cframe.use_tracing || opcode != LOAD_ATTR) {
}
else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *owner = TOP();
PyObject *name = GETITEM(names, oparg>>1);
next_instr--;
if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) {
goto error;
if (!cframe.use_tracing) {
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *owner = TOP();
PyObject *name = GETITEM(names, oparg>>1);
next_instr--;
if (_Py_Specialize_LoadAttr(owner, next_instr, name) < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(LOAD_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
}
Expand Down Expand Up @@ -2115,19 +2105,17 @@ dummy_func(

// stack effect: (__0, __1 -- )
inst(STORE_ATTR) {
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
if (cframe.use_tracing || opcode != STORE_ATTR) {
}
else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *owner = TOP();
PyObject *name = GETITEM(names, oparg);
next_instr--;
if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) {
goto error;
if (!cframe.use_tracing) {
_PyAttrCache *cache = (_PyAttrCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *owner = TOP();
PyObject *name = GETITEM(names, oparg);
next_instr--;
if (_Py_Specialize_StoreAttr(owner, next_instr, name) < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(STORE_ATTR, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
}
Expand Down Expand Up @@ -2247,26 +2235,24 @@ dummy_func(
}

inst(COMPARE_OP_GENERIC) {
GO_TO_INSTRUCTION(COMPARE_OP);
goto compare_op;
}

// stack effect: (__0 -- )
inst(COMPARE_OP) {
PREDICTED(COMPARE_OP);
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
if (cframe.use_tracing || opcode != COMPARE_OP) {
}
else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *right = TOP();
PyObject *left = SECOND();
next_instr--;
_Py_Specialize_CompareOp(left, right, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
else {
if (!cframe.use_tracing) {
_PyCompareOpCache *cache = (_PyCompareOpCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *right = TOP();
PyObject *left = SECOND();
next_instr--;
_Py_Specialize_CompareOp(left, right, next_instr, oparg);
DISPATCH_SAME_OPARG();
}
STAT_INC(COMPARE_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
}
compare_op:
assert(oparg <= Py_GE);
PyObject *right = POP();
PyObject *left = TOP();
Expand Down Expand Up @@ -2760,15 +2746,13 @@ dummy_func(

// stack effect: ( -- __0)
inst(FOR_ITER) {
_PyForIterCache *cache = (_PyForIterCache *)next_instr;
if (cframe.use_tracing || opcode != FOR_ITER) {
}
else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
next_instr--;
_Py_Specialize_ForIter(TOP(), next_instr);
DISPATCH_SAME_OPARG();
}
else {
if (!cframe.use_tracing) {
_PyForIterCache *cache = (_PyForIterCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
next_instr--;
_Py_Specialize_ForIter(TOP(), next_instr);
DISPATCH_SAME_OPARG();
}
STAT_INC(FOR_ITER, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
}
Expand Down Expand Up @@ -3085,22 +3069,20 @@ dummy_func(

// stack effect: (__0, __array[oparg] -- )
inst(CALL) {
_PyCallCache *cache = (_PyCallCache *)next_instr;
if (cframe.use_tracing || opcode != CALL) {
}
else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
next_instr--;
int is_meth = is_method(stack_pointer, oparg);
int nargs = oparg + is_meth;
PyObject *callable = PEEK(nargs + 1);
int err = _Py_Specialize_Call(callable, next_instr, nargs,
call_shape.kwnames);
if (err < 0) {
goto error;
if (!cframe.use_tracing) {
_PyCallCache *cache = (_PyCallCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
next_instr--;
int is_meth = is_method(stack_pointer, oparg);
int nargs = oparg + is_meth;
PyObject *callable = PEEK(nargs + 1);
int err = _Py_Specialize_Call(callable, next_instr, nargs,
call_shape.kwnames);
if (err < 0) {
goto error;
}
DISPATCH_SAME_OPARG();
}
DISPATCH_SAME_OPARG();
}
else {
STAT_INC(CALL, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
}
Expand Down Expand Up @@ -3875,26 +3857,24 @@ dummy_func(
}

inst(BINARY_OP_GENERIC) {
GO_TO_INSTRUCTION(BINARY_OP);
goto binary_op;
}

// stack effect: (__0 -- )
inst(BINARY_OP) {
PREDICTED(BINARY_OP);
_PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
if (cframe.use_tracing || opcode != BINARY_OP) {
}
else if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *lhs = SECOND();
PyObject *rhs = TOP();
next_instr--;
_Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0));
DISPATCH_SAME_OPARG();
}
else {
if (!cframe.use_tracing) {
_PyBinaryOpCache *cache = (_PyBinaryOpCache *)next_instr;
if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
PyObject *lhs = SECOND();
PyObject *rhs = TOP();
next_instr--;
_Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, &GETLOCAL(0));
DISPATCH_SAME_OPARG();
}
STAT_INC(BINARY_OP, deferred);
DECREMENT_ADAPTIVE_COUNTER(cache);
}
binary_op:
PyObject *rhs = POP();
PyObject *lhs = TOP();
assert(0 <= oparg);
Expand Down Expand Up @@ -3922,7 +3902,7 @@ dummy_func(
inst(EXTENDED_ARG) {
assert(oparg);
opcode = _Py_OPCODE(*next_instr);
if (cframe.use_tracing) {
if (!cframe.use_tracing) {
// Deoptimize the next opcode to avoid breaking tracing
// guarantees in quickened instructions:
opcode = _PyOpcode_Deopt[opcode];
Expand Down
8 changes: 0 additions & 8 deletions Python/ceval.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1251,14 +1251,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
STAT_INC(opcode, miss);
opcode = _PyOpcode_Deopt[opcode];
STAT_INC(opcode, miss);
/* The counter is always the first cache entry: */
_Py_CODEUNIT *counter = (_Py_CODEUNIT *)next_instr;
*counter -= 1;
if (*counter == 0) {
_Py_SET_OPCODE(next_instr[-1], opcode);
STAT_INC(opcode, deopt);
*counter = adaptive_counter_start();
}
next_instr--;
DISPATCH_GOTO();
}
Expand Down
Loading