bytecodes.c

// This file contains instruction definitions.
// It is read by Tools/cases_generator/generate_cases.py
// to generate Python/generated_cases.c.h.
// Note that there is some dummy C code at the top and bottom of the file
// to fool text editors like VS Code into believing this is valid C code.
// The actual instruction definitions start at // BEGIN BYTECODES //.
// See Tools/cases_generator/README.md for more information.

#include "Python.h"
#include "pycore_abstract.h"      // _PyIndex_Check()
#include "pycore_call.h"          // _PyObject_FastCallDictTstate()
#include "pycore_ceval.h"         // _PyEval_SignalAsyncExc()
#include "pycore_code.h"
#include "pycore_function.h"
#include "pycore_long.h"          // _PyLong_GetZero()
#include "pycore_object.h"        // _PyObject_GC_TRACK()
#include "pycore_moduleobject.h"  // PyModuleObject
#include "pycore_opcode.h"        // EXTRA_CASES
#include "pycore_pyerrors.h"      // _PyErr_Fetch()
#include "pycore_pymem.h"         // _PyMem_IsPtrFreed()
#include "pycore_pystate.h"       // _PyInterpreterState_GET()
#include "pycore_range.h"         // _PyRangeIterObject
#include "pycore_sliceobject.h"   // _PyBuildSlice_ConsumeRefs
#include "pycore_sysmodule.h"     // _PySys_Audit()
#include "pycore_tuple.h"         // _PyTuple_ITEMS()
#include "pycore_emscripten_signal.h"  // _Py_CHECK_EMSCRIPTEN_SIGNALS

#include "pycore_dict.h"
#include "dictobject.h"
#include "pycore_frame.h"
#include "opcode.h"
#include "pydtrace.h"
#include "setobject.h"
#include "structmember.h"         // struct PyMemberDef, T_OFFSET_EX

void _PyFloat_ExactDealloc(PyObject *);
void _PyUnicode_ExactDealloc(PyObject *);

/* Stack effect macros
 * These will be mostly replaced by stack effect descriptions,
 * but the tooling need to recognize them.
 */
#define SET_TOP(v)        (stack_pointer[-1] = (v))
#define SET_SECOND(v)     (stack_pointer[-2] = (v))
#define PEEK(n)           (stack_pointer[-(n)])
#define PUSH(val)         (*(stack_pointer++) = (val))
#define POP()             (*(--stack_pointer))
#define TOP()             PEEK(1)
#define SECOND()          PEEK(2)
#define STACK_GROW(n)     (stack_pointer += (n))
#define STACK_SHRINK(n)   (stack_pointer -= (n))
#define EMPTY()           1
#define STACK_LEVEL()     2

/* Local variable macros */
#define GETLOCAL(i)     (frame->localsplus[i])
#define SETLOCAL(i, val)  \
do { \
    PyObject *_tmp = frame->localsplus[i]; \
    frame->localsplus[i] = (val); \
    Py_XDECREF(_tmp); \
} while (0)

/* Flow control macros */
#define DEOPT_IF(cond, instname) ((void)0)
#define JUMPBY(offset) ((void)0)
#define GO_TO_INSTRUCTION(instname) ((void)0)
#define DISPATCH_SAME_OPARG() ((void)0)
#define DISPATCH() ((void)0)

#define inst(name) case name:
#define family(name) static int family_##name

#define NAME_ERROR_MSG \
    "name '%.200s' is not defined"

typedef struct {
    PyObject *kwnames;
} CallShape;

static PyObject *
dummy_func(
    PyThreadState *tstate,
    _PyInterpreterFrame *frame,
    unsigned char opcode,
    unsigned int oparg,
    _Py_atomic_int * const eval_breaker,
    _PyCFrame cframe,
    PyObject *names,
    PyObject *consts,
    _Py_CODEUNIT *next_instr,
    PyObject **stack_pointer,
    CallShape call_shape,
    _Py_CODEUNIT *first_instr,
    int throwflag,
    binaryfunc binary_ops[]
)
{
    switch (opcode) {

        /* BEWARE!
           It is essential that any operation that fails must goto error
           and that all operation that succeed call DISPATCH() ! */

// BEGIN BYTECODES //
        // stack effect: ( -- )
        inst(NOP) {
        }

        // stack effect: ( -- )
        inst(RESUME) {
            assert(tstate->cframe == &cframe);
            assert(frame == cframe.current_frame);
            if (_Py_atomic_load_relaxed_int32(eval_breaker) && oparg < 2) {
                goto handle_eval_breaker;
            }
        }

        // stack effect: ( -- __0)
        inst(LOAD_CLOSURE) {
            /* We keep LOAD_CLOSURE so that the bytecode stays more readable. */
            PyObject *value = GETLOCAL(oparg);
            if (value == NULL) {
                goto unbound_local_error;
            }
            Py_INCREF(value);
            PUSH(value);
        }

        // stack effect: ( -- __0)
        inst(LOAD_FAST_CHECK) {
            PyObject *value = GETLOCAL(oparg);
            if (value == NULL) {
                goto unbound_local_error;
            }
            Py_INCREF(value);
            PUSH(value);
        }

        // stack effect: ( -- __0)
        inst(LOAD_FAST) {
            PyObject *value = GETLOCAL(oparg);
            assert(value != NULL);
            Py_INCREF(value);
            PUSH(value);
        }

        // stack effect: ( -- __0)
        inst(LOAD_CONST) {
            PyObject *value = GETITEM(consts, oparg);
            Py_INCREF(value);
            PUSH(value);
        }

        // stack effect: (__0 -- )
        inst(STORE_FAST) {
            PyObject *value = POP();
            SETLOCAL(oparg, value);
        }

        // stack effect: ( -- __0, __1)
        inst(LOAD_FAST__LOAD_FAST) {
            PyObject *value = GETLOCAL(oparg);
            assert(value != NULL);
            NEXTOPARG();
            next_instr++;
            Py_INCREF(value);
            PUSH(value);
            value = GETLOCAL(oparg);
            assert(value != NULL);
            Py_INCREF(value);
            PUSH(value);
        }

        // stack effect: ( -- __0, __1)
        inst(LOAD_FAST__LOAD_CONST) {
            PyObject *value = GETLOCAL(oparg);
            assert(value != NULL);
            NEXTOPARG();
            next_instr++;
            Py_INCREF(value);
            PUSH(value);
            value = GETITEM(consts, oparg);
            Py_INCREF(value);
            PUSH(value);
        }

        // stack effect: ( -- )
        inst(STORE_FAST__LOAD_FAST) {
            PyObject *value = POP();
            SETLOCAL(oparg, value);
            NEXTOPARG();
            next_instr++;
            value = GETLOCAL(oparg);
            assert(value != NULL);
            Py_INCREF(value);
            PUSH(value);
        }

        // stack effect: (__0, __1 -- )
        inst(STORE_FAST__STORE_FAST) {
            PyObject *value = POP();
            SETLOCAL(oparg, value);
            NEXTOPARG();
            next_instr++;
            value = POP();
            SETLOCAL(oparg, value);
        }

        // stack effect: ( -- __0, __1)
        inst(LOAD_CONST__LOAD_FAST) {
            PyObject *value = GETITEM(consts, oparg);
            NEXTOPARG();
            next_instr++;
            Py_INCREF(value);
            PUSH(value);
            value = GETLOCAL(oparg);
            assert(value != NULL);
            Py_INCREF(value);
            PUSH(value);
        }

        // stack effect: (__0 -- )
        inst(POP_TOP) {
            PyObject *value = POP();
            Py_DECREF(value);
        }

        // stack effect: ( -- __0)
        inst(PUSH_NULL) {
            /* Use BASIC_PUSH as NULL is not a valid object pointer */
            BASIC_PUSH(NULL);
        }

        // stack effect: (__0, __1 -- )
        inst(END_FOR) {
            PyObject *value = POP();
            Py_DECREF(value);
            value = POP();
            Py_DECREF(value);
        }

        // stack effect: ( -- )
        inst(UNARY_POSITIVE) {
            PyObject *value = TOP();
            PyObject *res = PyNumber_Positive(value);
            Py_DECREF(value);
            SET_TOP(res);
            if (res == NULL)
                goto error;
        }

        // stack effect: ( -- )
        inst(UNARY_NEGATIVE) {
            PyObject *value = TOP();
            PyObject *res = PyNumber_Negative(value);
            Py_DECREF(value);
            SET_TOP(res);
            if (res == NULL)
                goto error;
        }

        // stack effect: ( -- )
        inst(UNARY_NOT) {
            PyObject *value = TOP();
            int err = PyObject_IsTrue(value);
            Py_DECREF(value);
            if (err == 0) {
                Py_INCREF(Py_True);
                SET_TOP(Py_True);
                DISPATCH();
            }
            else if (err > 0) {
                Py_INCREF(Py_False);
                SET_TOP(Py_False);
                DISPATCH();
            }
            STACK_SHRINK(1);
            goto error;
        }

        // stack effect: ( -- )
        inst(UNARY_INVERT) {
            PyObject *value = TOP();
            PyObject *res = PyNumber_Invert(value);
            Py_DECREF(value);
            SET_TOP(res);
            if (res == NULL)
                goto error;
        }

        // stack effect: (__0 -- )
        inst(BINARY_OP_MULTIPLY_INT) {
            assert(cframe.use_tracing == 0);
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
            DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
            SET_SECOND(prod);
            _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
            _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
            STACK_SHRINK(1);
            if (prod == NULL) {
                goto error;
            }
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
        }

        // stack effect: (__0 -- )
        inst(BINARY_OP_MULTIPLY_FLOAT) {
            assert(cframe.use_tracing == 0);
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
            DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            double dprod = ((PyFloatObject *)left)->ob_fval *
                ((PyFloatObject *)right)->ob_fval;
            PyObject *prod = PyFloat_FromDouble(dprod);
            SET_SECOND(prod);
            _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
            _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
            STACK_SHRINK(1);
            if (prod == NULL) {
                goto error;
            }
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
        }

        // stack effect: (__0 -- )
        inst(BINARY_OP_SUBTRACT_INT) {
            assert(cframe.use_tracing == 0);
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
            DEOPT_IF(!PyLong_CheckExact(right), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
            SET_SECOND(sub);
            _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
            _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
            STACK_SHRINK(1);
            if (sub == NULL) {
                goto error;
            }
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
        }

        // stack effect: (__0 -- )
        inst(BINARY_OP_SUBTRACT_FLOAT) {
            assert(cframe.use_tracing == 0);
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
            DEOPT_IF(!PyFloat_CheckExact(right), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval;
            PyObject *sub = PyFloat_FromDouble(dsub);
            SET_SECOND(sub);
            _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
            _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
            STACK_SHRINK(1);
            if (sub == NULL) {
                goto error;
            }
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
        }

        // stack effect: (__0 -- )
        inst(BINARY_OP_ADD_UNICODE) {
            assert(cframe.use_tracing == 0);
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
            DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            PyObject *res = PyUnicode_Concat(left, right);
            STACK_SHRINK(1);
            SET_TOP(res);
            _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
            _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
            if (TOP() == NULL) {
                goto error;
            }
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
        }

        // stack effect: (__0 -- )
        inst(BINARY_OP_INPLACE_ADD_UNICODE) {
            assert(cframe.use_tracing == 0);
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyUnicode_CheckExact(left), BINARY_OP);
            DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
            _Py_CODEUNIT true_next = next_instr[INLINE_CACHE_ENTRIES_BINARY_OP];
            assert(_Py_OPCODE(true_next) == STORE_FAST ||
                   _Py_OPCODE(true_next) == STORE_FAST__LOAD_FAST);
            PyObject **target_local = &GETLOCAL(_Py_OPARG(true_next));
            DEOPT_IF(*target_local != left, BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            /* Handle `left = left + right` or `left += right` for str.
             *
             * When possible, extend `left` in place rather than
             * allocating a new PyUnicodeObject. This attempts to avoid
             * quadratic behavior when one neglects to use str.join().
             *
             * If `left` has only two references remaining (one from
             * the stack, one in the locals), DECREFing `left` leaves
             * only the locals reference, so PyUnicode_Append knows
             * that the string is safe to mutate.
             */
            assert(Py_REFCNT(left) >= 2);
            _Py_DECREF_NO_DEALLOC(left);
            STACK_SHRINK(2);
            PyUnicode_Append(target_local, right);
            _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
            if (*target_local == NULL) {
                goto error;
            }
            // The STORE_FAST is already done.
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP + 1);
        }

        // stack effect: (__0 -- )
        inst(BINARY_OP_ADD_FLOAT) {
            assert(cframe.use_tracing == 0);
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyFloat_CheckExact(left), BINARY_OP);
            DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            double dsum = ((PyFloatObject *)left)->ob_fval +
                ((PyFloatObject *)right)->ob_fval;
            PyObject *sum = PyFloat_FromDouble(dsum);
            SET_SECOND(sum);
            _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
            _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
            STACK_SHRINK(1);
            if (sum == NULL) {
                goto error;
            }
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
        }

        // stack effect: (__0 -- )
        inst(BINARY_OP_ADD_INT) {
            assert(cframe.use_tracing == 0);
            PyObject *left = SECOND();
            PyObject *right = TOP();
            DEOPT_IF(!PyLong_CheckExact(left), BINARY_OP);
            DEOPT_IF(Py_TYPE(right) != Py_TYPE(left), BINARY_OP);
            STAT_INC(BINARY_OP, hit);
            PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
            SET_SECOND(sum);
            _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free);
            _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free);
            STACK_SHRINK(1);
            if (sum == NULL) {
                goto error;
            }
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_OP);
        }

        // stack effect: (__0 -- )
        inst(BINARY_SUBSCR) {
            PyObject *sub = POP();
            PyObject *container = TOP();
            PyObject *res = PyObject_GetItem(container, sub);
            Py_DECREF(container);
            Py_DECREF(sub);
            SET_TOP(res);
            if (res == NULL)
                goto error;
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
        }

        // stack effect: (__0, __1 -- )
        inst(BINARY_SLICE) {
            PyObject *stop = POP();
            PyObject *start = POP();
            PyObject *container = TOP();

            PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
            if (slice == NULL) {
                goto error;
            }
            PyObject *res = PyObject_GetItem(container, slice);
            Py_DECREF(slice);
            if (res == NULL) {
                goto error;
            }
            SET_TOP(res);
            Py_DECREF(container);
        }

        // stack effect: (__0, __1, __2, __3 -- )
        inst(STORE_SLICE) {
            PyObject *stop = POP();
            PyObject *start = POP();
            PyObject *container = TOP();
            PyObject *v = SECOND();

            PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop);
            if (slice == NULL) {
                goto error;
            }
            int err = PyObject_SetItem(container, slice, v);
            Py_DECREF(slice);
            if (err) {
                goto error;
            }
            STACK_SHRINK(2);
            Py_DECREF(v);
            Py_DECREF(container);
        }

        // stack effect: (__0 -- )
        inst(BINARY_SUBSCR_ADAPTIVE) {
            _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
            if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
                PyObject *sub = TOP();
                PyObject *container = SECOND();
                next_instr--;
                if (_Py_Specialize_BinarySubscr(container, sub, next_instr) < 0) {
                    goto error;
                }
                DISPATCH_SAME_OPARG();
            }
            else {
                STAT_INC(BINARY_SUBSCR, deferred);
                DECREMENT_ADAPTIVE_COUNTER(cache);
                GO_TO_INSTRUCTION(BINARY_SUBSCR);
            }
        }

        // stack effect: (__0 -- )
        inst(BINARY_SUBSCR_LIST_INT) {
            assert(cframe.use_tracing == 0);
            PyObject *sub = TOP();
            PyObject *list = SECOND();
            DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
            DEOPT_IF(!PyList_CheckExact(list), BINARY_SUBSCR);

            // Deopt unless 0 <= sub < PyList_Size(list)
            Py_ssize_t signed_magnitude = Py_SIZE(sub);
            DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR);
            assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0);
            Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
            DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            PyObject *res = PyList_GET_ITEM(list, index);
            assert(res != NULL);
            Py_INCREF(res);
            STACK_SHRINK(1);
            _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
            SET_TOP(res);
            Py_DECREF(list);
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
        }

        // stack effect: (__0 -- )
        inst(BINARY_SUBSCR_TUPLE_INT) {
            assert(cframe.use_tracing == 0);
            PyObject *sub = TOP();
            PyObject *tuple = SECOND();
            DEOPT_IF(!PyLong_CheckExact(sub), BINARY_SUBSCR);
            DEOPT_IF(!PyTuple_CheckExact(tuple), BINARY_SUBSCR);

            // Deopt unless 0 <= sub < PyTuple_Size(list)
            Py_ssize_t signed_magnitude = Py_SIZE(sub);
            DEOPT_IF(((size_t)signed_magnitude) > 1, BINARY_SUBSCR);
            assert(((PyLongObject *)_PyLong_GetZero())->ob_digit[0] == 0);
            Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
            DEOPT_IF(index >= PyTuple_GET_SIZE(tuple), BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            PyObject *res = PyTuple_GET_ITEM(tuple, index);
            assert(res != NULL);
            Py_INCREF(res);
            STACK_SHRINK(1);
            _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
            SET_TOP(res);
            Py_DECREF(tuple);
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
        }

        // stack effect: (__0 -- )
        inst(BINARY_SUBSCR_DICT) {
            assert(cframe.use_tracing == 0);
            PyObject *dict = SECOND();
            DEOPT_IF(!PyDict_CheckExact(SECOND()), BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            PyObject *sub = TOP();
            PyObject *res = PyDict_GetItemWithError(dict, sub);
            if (res == NULL) {
                if (!_PyErr_Occurred(tstate)) {
                    _PyErr_SetKeyError(sub);
                }
                goto error;
            }
            Py_INCREF(res);
            STACK_SHRINK(1);
            Py_DECREF(sub);
            SET_TOP(res);
            Py_DECREF(dict);
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
        }

        // stack effect: (__0 -- )
        inst(BINARY_SUBSCR_GETITEM) {
            PyObject *sub = TOP();
            PyObject *container = SECOND();
            _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)next_instr;
            uint32_t type_version = read_u32(cache->type_version);
            PyTypeObject *tp = Py_TYPE(container);
            DEOPT_IF(tp->tp_version_tag != type_version, BINARY_SUBSCR);
            assert(tp->tp_flags & Py_TPFLAGS_HEAPTYPE);
            PyObject *cached = ((PyHeapTypeObject *)tp)->_spec_cache.getitem;
            assert(PyFunction_Check(cached));
            PyFunctionObject *getitem = (PyFunctionObject *)cached;
            DEOPT_IF(getitem->func_version != cache->func_version, BINARY_SUBSCR);
            PyCodeObject *code = (PyCodeObject *)getitem->func_code;
            assert(code->co_argcount == 2);
            DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR);
            STAT_INC(BINARY_SUBSCR, hit);
            Py_INCREF(getitem);
            _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, getitem);
            STACK_SHRINK(2);
            new_frame->localsplus[0] = container;
            new_frame->localsplus[1] = sub;
            for (int i = 2; i < code->co_nlocalsplus; i++) {
                new_frame->localsplus[i] = NULL;
            }
            _PyFrame_SetStackPointer(frame, stack_pointer);
            JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
            frame->prev_instr = next_instr - 1;
            new_frame->previous = frame;
            frame = cframe.current_frame = new_frame;
            CALL_STAT_INC(inlined_py_calls);
            goto start_frame;
        }

        // stack effect: (__0 -- )
        inst(LIST_APPEND) {
            PyObject *v = POP();
            PyObject *list = PEEK(oparg);
            if (_PyList_AppendTakeRef((PyListObject *)list, v) < 0)
                goto error;
            PREDICT(JUMP_BACKWARD);
        }

        // stack effect: (__0 -- )
        inst(SET_ADD) {
            PyObject *v = POP();
            PyObject *set = PEEK(oparg);
            int err;
            err = PySet_Add(set, v);
            Py_DECREF(v);
            if (err != 0)
                goto error;
            PREDICT(JUMP_BACKWARD);
        }

        // stack effect: (__0, __1, __2 -- )
        inst(STORE_SUBSCR) {
            PyObject *sub = TOP();
            PyObject *container = SECOND();
            PyObject *v = THIRD();
            int err;
            STACK_SHRINK(3);
            /* container[sub] = v */
            err = PyObject_SetItem(container, sub, v);
            Py_DECREF(v);
            Py_DECREF(container);
            Py_DECREF(sub);
            if (err != 0) {
                goto error;
            }
            JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
        }

        // stack effect: (__0, __1, __2 -- )
        inst(STORE_SUBSCR_ADAPTIVE) {
            _PyStoreSubscrCache *cache = (_PyStoreSubscrCache *)next_instr;
            if (ADAPTIVE_COUNTER_IS_ZERO(cache)) {
                PyObject *sub = TOP();
                PyObject *container = SECOND();
                next_instr--;
                if (_Py_Specialize_StoreSubscr(container, sub, next_instr) < 0) {
                    goto error;
                }
                DISPATCH_SAME_OPARG();
            }
            else {
                STAT_INC(STORE_SUBSCR, deferred);
                DECREMENT_ADAPTIVE_COUNTER(cache);
                GO_TO_INSTRUCTION(STORE_SUBSCR);
            }
        }

        // stack effect: (__0, __1, __2 -- )
        inst(STORE_SUBSCR_LIST_INT) {
            assert(cframe.use_tracing == 0);
            PyObject *sub = TOP();
            PyObject *list = SECOND();
            PyObject *value = THIRD();
            DEOPT_IF(!PyLong_CheckExact(sub), STORE_SUBSCR);
            DEOPT_IF(!PyList_CheckExact(list), STORE_SUBSCR);

            // Ensure nonnegative, zero-or-one-digit ints.
            DEOPT_IF(((size_t)Py_SIZE(sub)) > 1, STORE_SUBSCR);
            Py_ssize_t index = ((PyLongObject*)sub)->ob_digit[0];
            // Ensure index < len(list)
            DEOPT_IF(index >= PyList_GET_SIZE(list), STORE_SUBSCR);
            STAT_INC(STORE_SUBSCR, hit);

            PyObject *old_value = PyList_GET_ITEM(list, index);
            PyList_SET_ITEM(list, index, value);
            STACK_SHRINK(3);
            assert(old_value != NULL);
            Py_DECREF(old_value);
            _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free);
            Py_DECREF(list);
            JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
        }

        // stack effect: (__0, __1, __2 -- )
        inst(STORE_SUBSCR_DICT) {
            assert(cframe.use_tracing == 0);
            PyObject *sub = TOP();
            PyObject *dict = SECOND();
            PyObject *value = THIRD();
            DEOPT_IF(!PyDict_CheckExact(dict), STORE_SUBSCR);
            STACK_SHRINK(3);
            STAT_INC(STORE_SUBSCR, hit);
            int err = _PyDict_SetItem_Take2((PyDictObject *)dict, sub, value);
            Py_DECREF(dict);
            if (err != 0) {
                goto error;
            }
            JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
        }

        // stack effect: (__0, __1 -- )
        inst(DELETE_SUBSCR) {
            PyObject *sub = TOP();
            PyObject *container = SECOND();
            int err;
            STACK_SHRINK(2);
            /* del container[sub] */
            err = PyObject_DelItem(container, sub);
            Py_DECREF(container);
            Py_DECREF(sub);
            if (err != 0)
                goto error;
        }

        // stack effect: (__0 -- )
        inst(PRINT_EXPR) {
            PyObject *value = POP();
            PyObject *hook = _PySys_GetAttr(tstate, &_Py_ID(displayhook));
            PyObject *res;
            if (hook == NULL) {
                _PyErr_SetString(tstate, PyExc_RuntimeError,
                                 "lost sys.displayhook");
                Py_DECREF(value);
                goto error;
            }
            res = PyObject_CallOneArg(hook, value);
            Py_DECREF(value);
            if (res == NULL)
                goto error;
            Py_DECREF(res);
        }

        // stack effect: (__array[oparg] -- )
        inst(RAISE_VARARGS) {
            PyObject *cause = NULL, *exc = NULL;
            switch (oparg) {
            case 2:
                cause = POP(); /* cause */
                /* fall through */
            case 1:
                exc = POP(); /* exc */
                /* fall through */
            case 0:
                if (do_raise(tstate, exc, cause)) {
                    goto exception_unwind;
                }
                break;
            default:
                _PyErr_SetString(tstate, PyExc_SystemError,
                                 "bad RAISE_VARARGS oparg");
                break;
            }
            goto error;
        }

        // stack effect: (__0 -- )
        inst(RETURN_VALUE) {
            PyObject *retval = POP();
            assert(EMPTY());
            _PyFrame_SetStackPointer(frame, stack_pointer);
            TRACE_FUNCTION_EXIT();
            DTRACE_FUNCTION_EXIT();
            _Py_LeaveRecursiveCallPy(tstate);
            if (!frame->is_entry) {
                frame = cframe.current_frame = pop_frame(tstate, frame);
                _PyFrame_StackPush(frame, retval);
                goto resume_frame;
            }
            _Py_LeaveRecursiveCallTstate(tstate);
            /* Restore previous cframe and return. */
            tstate->cframe = cframe.previous;
            tstate->cframe->use_tracing = cframe.use_tracing;
            assert(tstate->cframe->current_frame == frame->previous);
            assert(!_PyErr_Occurred(tstate));
            return retval;
        }

        // stack effect: ( -- )
        inst(GET_AITER) {
            unaryfunc getter = NULL;
            PyObject *iter = NULL;
            PyObject *obj = TOP();
            PyTypeObject *type = Py_TYPE(obj);

            if (type->tp_as_async != NULL) {
                getter = type->tp_as_async->am_aiter;
            }

            if (getter != NULL) {
                iter = (*getter)(obj);
                Py_DECREF(obj);
                if (iter == NULL) {
                    SET_TOP(NULL);
                    goto error;
                }
            }
            else {
                SET_TOP(NULL);
                _PyErr_Format(tstate, PyExc_TypeError,
                              "'async for' requires an object with "
                              "__aiter__ method, got %.100s",
                              type->tp_name);
                Py_DECREF(obj);
                goto error;
            }

            if (Py_TYPE(iter)->tp_as_async == NULL ||
                    Py_TYPE(iter)->tp_as_async->am_anext == NULL) {

                SET_TOP(NULL);
                _PyErr_Format(tstate, PyExc_TypeError,
                              "'async for' received an object from __aiter__ "
                              "that does not implement __anext__: %.100s",
                              Py_TYPE(iter)->tp_name);
                Py_DECREF(iter);
                goto error;
            }

            SET_TOP(iter);
        }

        // stack effect: ( -- __0)
        inst(GET_ANEXT) {
            unaryfunc getter = NULL;
            PyObject *next_iter = NULL;
            PyObject *awaitable = NULL;
            PyObject *aiter = TOP();
            PyTypeObject *type = Py_TYPE(aiter);

            if (PyAsyncGen_CheckExact(aiter)) {
                awaitable = type->tp_as_async->am_anext(aiter);
                if (awaitable == NULL) {
                    goto error;
                }
            } else {
                if (type->tp_as_async != NULL){
                    getter = type->tp_as_async->am_anext;
                }

                if (getter != NULL) {
                    next_iter = (*getter)(aiter);
                    if (next_iter == NULL) {
                        goto error;
                    }
                }
                else {
                    _PyErr_Format(tstate, PyExc_TypeError,
                                  "'async for' requires an iterator with "
                                  "__anext__ method, got %.100s",
                                  type->tp_name);
                    goto error;
                }

                awaitable = _PyCoro_GetAwaitableIter(next_iter);
                if (awaitable == NULL) {
                    _PyErr_FormatFromCause(
                        PyExc_TypeError,
                        "'async for' received an invalid object "
                        "from __anext__: %.100s",
                        Py_TYPE(next_iter)->tp_name);

                    Py_DECREF(next_iter);
                    goto error;
                } else {
                    Py_DECREF(next_iter);
                }
            }

            PUSH(awaitable);
            PREDICT(LOAD_CONST);
        }

        // stack effect: ( -- )
        inst(GET_AWAITABLE) {
            PyObject *iterable = TOP();
            PyObject *iter = _PyCoro_GetAwaitableIter(iterable);

            if (iter == NULL) {
                format_awaitable_error(tstate, Py_TYPE(iterable), oparg);
            }

            Py_DECREF(iterable);

            if (iter != NULL && PyCoro_CheckExact(iter)) {
                PyObject *yf = _PyGen_yf((PyGenObject*)iter);
                if (yf != NULL) {
                    /* `iter` is a coroutine object that is being
                       awaited, `yf` is a pointer to the current awaitable
                       being awaited on. */
                    Py_DECREF(yf);
                    Py_CLEAR(iter);
                    _PyErr_SetString(tstate, PyExc_RuntimeError,
                                     "coroutine is being awaited already");
                    /* The code below jumps to `error` if `iter` is NULL. */
                }
            }

            SET_TOP(iter); /* Even if it's NULL */

            if (iter == NULL) {
                goto error;
            }

            PREDICT(LOAD_CONST);
        }

        // error: SEND stack effect depends on jump flag
        inst(SEND) {
            assert(frame->is_entry);
            assert(STACK_LEVEL() >= 2);
            PyObject *v = POP();
            PyObject *receiver = TOP();
            PySendResult gen_status;
            PyObject *retval;
            if (tstate->c_tracefunc == NULL) {
                gen_status = PyIter_Send(receiver, v, &retval);
            } else {
                if (Py_IsNone(v) && PyIter_Check(receiver)) {
                    retval = Py_TYPE(receiver)->tp_iternext(receiver);
                }
                else {
                    retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v);
                }
                if (retval == NULL) {
                    if (tstate->c_tracefunc != NULL
                            && _PyErr_ExceptionMatches(tstate, PyExc_StopIteration))
                        call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame);
                    if (_PyGen_FetchStopIterationValue(&retval) == 0) {
                        gen_status = PYGEN_RETURN;
                    }
                    else {
                        gen_status = PYGEN_ERROR;
                    }
                }
                else {
                    gen_status = PYGEN_NEXT;
                }
            }
            Py_DECREF(v);
            if (gen_status == PYGEN_ERROR) {
                assert(retval == NULL);
                goto error;
            }
            if (gen_status == PYGEN_RETURN) {
                assert(retval != NULL);
                Py_DECREF(receiver);
                SET_TOP(retval);
                JUMPBY(oparg);
                DISPATCH();
            }
            assert(gen_status == PYGEN_NEXT);
            assert(retval != NULL);
            PUSH(retval);
        }

        // stack effect: ( -- )