specialize.c

#include "Python.h"
#include "pycore_code.h"
#include "pycore_dict.h"
#include "pycore_function.h"      // _PyFunction_GetVersionForCurrentState()
#include "pycore_global_strings.h"  // _Py_ID()
#include "pycore_long.h"
#include "pycore_moduleobject.h"
#include "pycore_object.h"
#include "opcode.h"
#include "structmember.h"         // struct PyMemberDef, T_OFFSET_EX

#include <stdlib.h> // rand()

/* For guidance on adding or extending families of instructions see
 * ./adaptive.md
 */


/* We layout the quickened data as a bi-directional array:
 * Instructions upwards, cache entries downwards.
 * first_instr is aligned to a SpecializedCacheEntry.
 * The nth instruction is located at first_instr[n]
 * The nth cache is located at ((SpecializedCacheEntry *)first_instr)[-1-n]
 * The first (index 0) cache entry is reserved for the count, to enable finding
 * the first instruction from the base pointer.
 * The cache_count argument must include space for the count.
 * We use the SpecializedCacheOrInstruction union to refer to the data
 * to avoid type punning.

 Layout of quickened data, each line 8 bytes for M cache entries and N instructions:

 <cache_count>                              <---- co->co_quickened
 <cache M-1>
 <cache M-2>
 ...
 <cache 0>
 <instr 0> <instr 1> <instr 2> <instr 3>    <--- co->co_first_instr
 <instr 4> <instr 5> <instr 6> <instr 7>
 ...
 <instr N-1>
*/

/* Map from opcode to adaptive opcode.
  Values of zero are ignored. */
static uint8_t adaptive_opcodes[256] = {
    [LOAD_ATTR] = LOAD_ATTR_ADAPTIVE,
    [LOAD_GLOBAL] = LOAD_GLOBAL_ADAPTIVE,
    [LOAD_METHOD] = LOAD_METHOD_ADAPTIVE,
    [BINARY_SUBSCR] = BINARY_SUBSCR_ADAPTIVE,
    [STORE_SUBSCR] = STORE_SUBSCR_ADAPTIVE,
    [CALL] = CALL_ADAPTIVE,
    [PRECALL] = PRECALL_ADAPTIVE,
    [STORE_ATTR] = STORE_ATTR_ADAPTIVE,
    [BINARY_OP] = BINARY_OP_ADAPTIVE,
    [COMPARE_OP] = COMPARE_OP_ADAPTIVE,
    [UNPACK_SEQUENCE] = UNPACK_SEQUENCE_ADAPTIVE,
};

Py_ssize_t _Py_QuickenedCount = 0;
#ifdef Py_STATS
PyStats _py_stats = { 0 };

#define ADD_STAT_TO_DICT(res, field) \
    do { \
        PyObject *val = PyLong_FromUnsignedLongLong(stats->field); \
        if (val == NULL) { \
            Py_DECREF(res); \
            return NULL; \
        } \
        if (PyDict_SetItemString(res, #field, val) == -1) { \
            Py_DECREF(res); \
            Py_DECREF(val); \
            return NULL; \
        } \
        Py_DECREF(val); \
    } while(0);

static PyObject*
stats_to_dict(SpecializationStats *stats)
{
    PyObject *res = PyDict_New();
    if (res == NULL) {
        return NULL;
    }
    ADD_STAT_TO_DICT(res, success);
    ADD_STAT_TO_DICT(res, failure);
    ADD_STAT_TO_DICT(res, hit);
    ADD_STAT_TO_DICT(res, deferred);
    ADD_STAT_TO_DICT(res, miss);
    ADD_STAT_TO_DICT(res, deopt);
    PyObject *failure_kinds = PyTuple_New(SPECIALIZATION_FAILURE_KINDS);
    if (failure_kinds == NULL) {
        Py_DECREF(res);
        return NULL;
    }
    for (int i = 0; i < SPECIALIZATION_FAILURE_KINDS; i++) {
        PyObject *stat = PyLong_FromUnsignedLongLong(stats->failure_kinds[i]);
        if (stat == NULL) {
            Py_DECREF(res);
            Py_DECREF(failure_kinds);
            return NULL;
        }
        PyTuple_SET_ITEM(failure_kinds, i, stat);
    }
    if (PyDict_SetItemString(res, "failure_kinds", failure_kinds)) {
        Py_DECREF(res);
        Py_DECREF(failure_kinds);
        return NULL;
    }
    Py_DECREF(failure_kinds);
    return res;
}
#undef ADD_STAT_TO_DICT

static int
add_stat_dict(
    PyObject *res,
    int opcode,
    const char *name) {

    SpecializationStats *stats = &_py_stats.opcode_stats[opcode].specialization;
    PyObject *d = stats_to_dict(stats);
    if (d == NULL) {
        return -1;
    }
    int err = PyDict_SetItemString(res, name, d);
    Py_DECREF(d);
    return err;
}

#ifdef Py_STATS
PyObject*
_Py_GetSpecializationStats(void) {
    PyObject *stats = PyDict_New();
    if (stats == NULL) {
        return NULL;
    }
    int err = 0;
    err += add_stat_dict(stats, LOAD_ATTR, "load_attr");
    err += add_stat_dict(stats, LOAD_GLOBAL, "load_global");
    err += add_stat_dict(stats, LOAD_METHOD, "load_method");
    err += add_stat_dict(stats, BINARY_SUBSCR, "binary_subscr");
    err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr");
    err += add_stat_dict(stats, STORE_ATTR, "store_attr");
    err += add_stat_dict(stats, CALL, "call");
    err += add_stat_dict(stats, BINARY_OP, "binary_op");
    err += add_stat_dict(stats, COMPARE_OP, "compare_op");
    err += add_stat_dict(stats, UNPACK_SEQUENCE, "unpack_sequence");
    err += add_stat_dict(stats, PRECALL, "precall");
    if (err < 0) {
        Py_DECREF(stats);
        return NULL;
    }
    return stats;
}
#endif


#define PRINT_STAT(i, field) \
    if (stats[i].field) { \
        fprintf(out, "    opcode[%d]." #field " : %" PRIu64 "\n", i, stats[i].field); \
    }

static void
print_spec_stats(FILE *out, OpcodeStats *stats)
{
    /* Mark some opcodes as specializable for stats,
     * even though we don't specialize them yet. */
    fprintf(out, "opcode[%d].specializable : 1\n", FOR_ITER);
    for (int i = 0; i < 256; i++) {
        if (adaptive_opcodes[i]) {
            fprintf(out, "opcode[%d].specializable : 1\n", i);
        }
        PRINT_STAT(i, specialization.success);
        PRINT_STAT(i, specialization.failure);
        PRINT_STAT(i, specialization.hit);
        PRINT_STAT(i, specialization.deferred);
        PRINT_STAT(i, specialization.miss);
        PRINT_STAT(i, specialization.deopt);
        PRINT_STAT(i, execution_count);
        for (int j = 0; j < SPECIALIZATION_FAILURE_KINDS; j++) {
            uint64_t val = stats[i].specialization.failure_kinds[j];
            if (val) {
                fprintf(out, "    opcode[%d].specialization.failure_kinds[%d] : %"
                    PRIu64 "\n", i, j, val);
            }
        }
        for(int j = 0; j < 256; j++) {
            if (stats[i].pair_count[j]) {
                fprintf(out, "opcode[%d].pair_count[%d] : %" PRIu64 "\n",
                        i, j, stats[i].pair_count[j]);
            }
        }
    }
}
#undef PRINT_STAT


static void
print_call_stats(FILE *out, CallStats *stats)
{
    fprintf(out, "Calls to PyEval_EvalDefault: %" PRIu64 "\n", stats->pyeval_calls);
    fprintf(out, "Calls to Python functions inlined: %" PRIu64 "\n", stats->inlined_py_calls);
    fprintf(out, "Frames pushed: %" PRIu64 "\n", stats->frames_pushed);
    fprintf(out, "Frame objects created: %" PRIu64 "\n", stats->frame_objects_created);
}

static void
print_object_stats(FILE *out, ObjectStats *stats)
{
    fprintf(out, "Object allocations: %" PRIu64 "\n", stats->allocations);
    fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
    fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
    fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
    fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
    fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);
    fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass);
}

static void
print_stats(FILE *out, PyStats *stats) {
    print_spec_stats(out, stats->opcode_stats);
    print_call_stats(out, &stats->call_stats);
    print_object_stats(out, &stats->object_stats);
}

void
_Py_PrintSpecializationStats(int to_file)
{
    FILE *out = stderr;
    if (to_file) {
        /* Write to a file instead of stderr. */
# ifdef MS_WINDOWS
        const char *dirname = "c:\\temp\\py_stats\\";
# else
        const char *dirname = "/tmp/py_stats/";
# endif
        /* Use random 160 bit number as file name,
        * to avoid both accidental collisions and
        * symlink attacks. */
        unsigned char rand[20];
        char hex_name[41];
        _PyOS_URandomNonblock(rand, 20);
        for (int i = 0; i < 20; i++) {
            hex_name[2*i] = "0123456789abcdef"[rand[i]&15];
            hex_name[2*i+1] = "0123456789abcdef"[(rand[i]>>4)&15];
        }
        hex_name[40] = '\0';
        char buf[64];
        assert(strlen(dirname) + 40 + strlen(".txt") < 64);
        sprintf(buf, "%s%s.txt", dirname, hex_name);
        FILE *fout = fopen(buf, "w");
        if (fout) {
            out = fout;
        }
    }
    else {
        fprintf(out, "Specialization stats:\n");
    }
    print_stats(out, &_py_stats);
    if (out != stderr) {
        fclose(out);
    }
}

#ifdef Py_STATS

#define SPECIALIZATION_FAIL(opcode, kind) _py_stats.opcode_stats[opcode].specialization.failure_kinds[kind]++


#endif
#endif

#ifndef SPECIALIZATION_FAIL
#define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
#endif

static _Py_CODEUNIT *
allocate(int instruction_count)
{
    assert(instruction_count > 0);
    void *array = PyMem_Malloc(sizeof(_Py_CODEUNIT) * instruction_count);
    if (array == NULL) {
        PyErr_NoMemory();
        return NULL;
    }
    _Py_QuickenedCount++;
    return (_Py_CODEUNIT *)array;
}


// Insert adaptive instructions and superinstructions.
static void
optimize(_Py_CODEUNIT *instructions, int len)
{
    int previous_opcode = -1;
    for(int i = 0; i < len; i++) {
        int opcode = _Py_OPCODE(instructions[i]);
        uint8_t adaptive_opcode = adaptive_opcodes[opcode];
        if (adaptive_opcode) {
            _Py_SET_OPCODE(instructions[i], adaptive_opcode);
            // Make sure the adaptive counter is zero:
            assert(instructions[i + 1] == 0);
            previous_opcode = -1;
            i += _PyOpcode_InlineCacheEntries[opcode];
        }
        else {
            assert(!_PyOpcode_InlineCacheEntries[opcode]);
            switch (opcode) {
                case JUMP_ABSOLUTE:
                    _Py_SET_OPCODE(instructions[i], JUMP_ABSOLUTE_QUICK);
                    break;
                case RESUME:
                    _Py_SET_OPCODE(instructions[i], RESUME_QUICK);
                    break;
                case LOAD_FAST:
                    switch(previous_opcode) {
                        case LOAD_FAST:
                            _Py_SET_OPCODE(instructions[i - 1],
                                           LOAD_FAST__LOAD_FAST);
                            break;
                        case STORE_FAST:
                            _Py_SET_OPCODE(instructions[i - 1],
                                           STORE_FAST__LOAD_FAST);
                            break;
                        case LOAD_CONST:
                            _Py_SET_OPCODE(instructions[i - 1],
                                           LOAD_CONST__LOAD_FAST);
                            break;
                    }
                    break;
                case STORE_FAST:
                    if (previous_opcode == STORE_FAST) {
                        _Py_SET_OPCODE(instructions[i - 1],
                                       STORE_FAST__STORE_FAST);
                    }
                    break;
                case LOAD_CONST:
                    if (previous_opcode == LOAD_FAST) {
                        _Py_SET_OPCODE(instructions[i - 1],
                                       LOAD_FAST__LOAD_CONST);
                    }
                    break;
            }
            previous_opcode = opcode;
        }
    }
}

int
_Py_Quicken(PyCodeObject *code) {
    if (code->co_quickened) {
        return 0;
    }
    Py_ssize_t size = PyBytes_GET_SIZE(code->co_code);
    int instr_count = (int)(size/sizeof(_Py_CODEUNIT));
    if (instr_count > MAX_SIZE_TO_QUICKEN) {
        code->co_warmup = QUICKENING_WARMUP_COLDEST;
        return 0;
    }
    _Py_CODEUNIT *quickened = allocate(instr_count);
    if (quickened == NULL) {
        return -1;
    }
    memcpy(quickened, code->co_firstinstr, size);
    optimize(quickened, instr_count);
    code->co_quickened = quickened;
    code->co_firstinstr = quickened;
    return 0;
}

static inline int
initial_counter_value(void) {
    /* Starting value for the counter.
     * This value needs to be not too low, otherwise
     * it would cause excessive de-optimization.
     * Neither should it be too high, or that would delay
     * de-optimization excessively when it is needed.
     * A value around 50 seems to work, and we choose a
     * prime number to avoid artifacts.
     */
    return 53;
}

/* Common */

#define SPEC_FAIL_OTHER 0
#define SPEC_FAIL_NO_DICT 1
#define SPEC_FAIL_OVERRIDDEN 2
#define SPEC_FAIL_OUT_OF_VERSIONS 3
#define SPEC_FAIL_OUT_OF_RANGE 4
#define SPEC_FAIL_EXPECTED_ERROR 5
#define SPEC_FAIL_WRONG_NUMBER_ARGUMENTS 6

#define SPEC_FAIL_LOAD_GLOBAL_NON_STRING_OR_SPLIT 18

/* Attributes */

#define SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR 8
#define SPEC_FAIL_ATTR_NON_OVERRIDING_DESCRIPTOR 9
#define SPEC_FAIL_ATTR_NOT_DESCRIPTOR 10
#define SPEC_FAIL_ATTR_METHOD 11
#define SPEC_FAIL_ATTR_MUTABLE_CLASS 12
#define SPEC_FAIL_ATTR_PROPERTY 13
#define SPEC_FAIL_ATTR_NON_OBJECT_SLOT 14
#define SPEC_FAIL_ATTR_READ_ONLY 15
#define SPEC_FAIL_ATTR_AUDITED_SLOT 16
#define SPEC_FAIL_ATTR_NOT_MANAGED_DICT 17
#define SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT 18
#define SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND 19

/* Methods */

#define SPEC_FAIL_LOAD_METHOD_OVERRIDING_DESCRIPTOR 8
#define SPEC_FAIL_LOAD_METHOD_NON_OVERRIDING_DESCRIPTOR 9
#define SPEC_FAIL_LOAD_METHOD_NOT_DESCRIPTOR 10
#define SPEC_FAIL_LOAD_METHOD_METHOD 11
#define SPEC_FAIL_LOAD_METHOD_MUTABLE_CLASS 12
#define SPEC_FAIL_LOAD_METHOD_PROPERTY 13
#define SPEC_FAIL_LOAD_METHOD_NON_OBJECT_SLOT 14
#define SPEC_FAIL_LOAD_METHOD_IS_ATTR 15
#define SPEC_FAIL_LOAD_METHOD_DICT_SUBCLASS 16
#define SPEC_FAIL_LOAD_METHOD_BUILTIN_CLASS_METHOD 17
#define SPEC_FAIL_LOAD_METHOD_CLASS_METHOD_OBJ 18
#define SPEC_FAIL_LOAD_METHOD_OBJECT_SLOT 19
#define SPEC_FAIL_LOAD_METHOD_HAS_DICT 20
#define SPEC_FAIL_LOAD_METHOD_HAS_MANAGED_DICT 21
#define SPEC_FAIL_LOAD_METHOD_INSTANCE_ATTRIBUTE 22
#define SPEC_FAIL_LOAD_METHOD_METACLASS_ATTRIBUTE 23

/* Binary subscr and store subscr */

#define SPEC_FAIL_SUBSCR_ARRAY_INT 8
#define SPEC_FAIL_SUBSCR_ARRAY_SLICE 9
#define SPEC_FAIL_SUBSCR_LIST_SLICE 10
#define SPEC_FAIL_SUBSCR_TUPLE_SLICE 11
#define SPEC_FAIL_SUBSCR_STRING_INT 12
#define SPEC_FAIL_SUBSCR_STRING_SLICE 13
#define SPEC_FAIL_SUBSCR_BUFFER_INT 15
#define SPEC_FAIL_SUBSCR_BUFFER_SLICE 16
#define SPEC_FAIL_SUBSCR_SEQUENCE_INT 17

/* Store subscr */
#define SPEC_FAIL_SUBSCR_BYTEARRAY_INT 18
#define SPEC_FAIL_SUBSCR_BYTEARRAY_SLICE 19
#define SPEC_FAIL_SUBSCR_PY_SIMPLE 20
#define SPEC_FAIL_SUBSCR_PY_OTHER 21
#define SPEC_FAIL_SUBSCR_DICT_SUBCLASS_NO_OVERRIDE 22
#define SPEC_FAIL_SUBSCR_NOT_HEAP_TYPE 23

/* Binary op */

#define SPEC_FAIL_BINARY_OP_ADD_DIFFERENT_TYPES          8
#define SPEC_FAIL_BINARY_OP_ADD_OTHER                    9
#define SPEC_FAIL_BINARY_OP_AND_DIFFERENT_TYPES         10
#define SPEC_FAIL_BINARY_OP_AND_INT                     11
#define SPEC_FAIL_BINARY_OP_AND_OTHER                   12
#define SPEC_FAIL_BINARY_OP_FLOOR_DIVIDE                13
#define SPEC_FAIL_BINARY_OP_LSHIFT                      14
#define SPEC_FAIL_BINARY_OP_MATRIX_MULTIPLY             15
#define SPEC_FAIL_BINARY_OP_MULTIPLY_DIFFERENT_TYPES    16
#define SPEC_FAIL_BINARY_OP_MULTIPLY_OTHER              17
#define SPEC_FAIL_BINARY_OP_OR                          18
#define SPEC_FAIL_BINARY_OP_POWER                       19
#define SPEC_FAIL_BINARY_OP_REMAINDER                   20
#define SPEC_FAIL_BINARY_OP_RSHIFT                      21
#define SPEC_FAIL_BINARY_OP_SUBTRACT_DIFFERENT_TYPES    22
#define SPEC_FAIL_BINARY_OP_SUBTRACT_OTHER              23
#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_DIFFERENT_TYPES 24
#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_FLOAT           25
#define SPEC_FAIL_BINARY_OP_TRUE_DIVIDE_OTHER           26
#define SPEC_FAIL_BINARY_OP_XOR                         27

/* Calls */
#define SPEC_FAIL_CALL_COMPLEX_PARAMETERS 9
#define SPEC_FAIL_CALL_CO_NOT_OPTIMIZED 10
/* SPEC_FAIL_METHOD  defined as 11 above */

#define SPEC_FAIL_CALL_INSTANCE_METHOD 11
#define SPEC_FAIL_CALL_CMETHOD 12
#define SPEC_FAIL_CALL_PYCFUNCTION 13
#define SPEC_FAIL_CALL_PYCFUNCTION_WITH_KEYWORDS 14
#define SPEC_FAIL_CALL_PYCFUNCTION_FAST_WITH_KEYWORDS 15
#define SPEC_FAIL_CALL_PYCFUNCTION_NOARGS 16
#define SPEC_FAIL_CALL_BAD_CALL_FLAGS 17
#define SPEC_FAIL_CALL_CLASS 18
#define SPEC_FAIL_CALL_PYTHON_CLASS 19
#define SPEC_FAIL_CALL_METHOD_DESCRIPTOR 20
#define SPEC_FAIL_CALL_BOUND_METHOD 21
#define SPEC_FAIL_CALL_STR 22
#define SPEC_FAIL_CALL_CLASS_NO_VECTORCALL 23
#define SPEC_FAIL_CALL_CLASS_MUTABLE 24
#define SPEC_FAIL_CALL_KWNAMES 25
#define SPEC_FAIL_CALL_METHOD_WRAPPER 26
#define SPEC_FAIL_CALL_OPERATOR_WRAPPER 27
#define SPEC_FAIL_CALL_PYFUNCTION 28

/* COMPARE_OP */
#define SPEC_FAIL_COMPARE_OP_DIFFERENT_TYPES 12
#define SPEC_FAIL_COMPARE_OP_STRING 13
#define SPEC_FAIL_COMPARE_OP_NOT_FOLLOWED_BY_COND_JUMP 14
#define SPEC_FAIL_COMPARE_OP_BIG_INT 15
#define SPEC_FAIL_COMPARE_OP_BYTES 16
#define SPEC_FAIL_COMPARE_OP_TUPLE 17
#define SPEC_FAIL_COMPARE_OP_LIST 18
#define SPEC_FAIL_COMPARE_OP_SET 19
#define SPEC_FAIL_COMPARE_OP_BOOL 20
#define SPEC_FAIL_COMPARE_OP_BASEOBJECT 21
#define SPEC_FAIL_COMPARE_OP_FLOAT_LONG 22
#define SPEC_FAIL_COMPARE_OP_LONG_FLOAT 23
#define SPEC_FAIL_COMPARE_OP_EXTENDED_ARG 24

/* FOR_ITER */
#define SPEC_FAIL_FOR_ITER_GENERATOR 10
#define SPEC_FAIL_FOR_ITER_COROUTINE 11
#define SPEC_FAIL_FOR_ITER_ASYNC_GENERATOR 12
#define SPEC_FAIL_FOR_ITER_LIST 13
#define SPEC_FAIL_FOR_ITER_TUPLE 14
#define SPEC_FAIL_FOR_ITER_SET 15
#define SPEC_FAIL_FOR_ITER_STRING 16
#define SPEC_FAIL_FOR_ITER_BYTES 17
#define SPEC_FAIL_FOR_ITER_RANGE 18
#define SPEC_FAIL_FOR_ITER_ITERTOOLS 19
#define SPEC_FAIL_FOR_ITER_DICT_KEYS 20
#define SPEC_FAIL_FOR_ITER_DICT_ITEMS 21
#define SPEC_FAIL_FOR_ITER_DICT_VALUES 22
#define SPEC_FAIL_FOR_ITER_ENUMERATE 23

// UNPACK_SEQUENCE

#define SPEC_FAIL_UNPACK_SEQUENCE_ITERATOR 8
#define SPEC_FAIL_UNPACK_SEQUENCE_SEQUENCE 9


static int
specialize_module_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
                            PyObject *name, int opcode, int opcode_module)
{
    _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
    PyModuleObject *m = (PyModuleObject *)owner;
    PyObject *value = NULL;
    assert((owner->ob_type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0);
    PyDictObject *dict = (PyDictObject *)m->md_dict;
    if (dict == NULL) {
        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_NO_DICT);
        return -1;
    }
    if (dict->ma_keys->dk_kind != DICT_KEYS_UNICODE) {
        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_NON_STRING_OR_SPLIT);
        return -1;
    }
    Py_ssize_t index = _PyDict_GetItemHint(dict, &_Py_ID(__getattr__), -1,
                                           &value);
    assert(index != DKIX_ERROR);
    if (index != DKIX_EMPTY) {
        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_ATTR_MODULE_ATTR_NOT_FOUND);
        return -1;
    }
    index = _PyDict_GetItemHint(dict, name, -1, &value);
    assert (index != DKIX_ERROR);
    if (index != (uint16_t)index) {
        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_RANGE);
        return -1;
    }
    uint32_t keys_version = _PyDictKeys_GetVersionForCurrentState(dict->ma_keys);
    if (keys_version == 0) {
        SPECIALIZATION_FAIL(opcode, SPEC_FAIL_OUT_OF_VERSIONS);
        return -1;
    }
    write_u32(cache->version, keys_version);
    cache->index = (uint16_t)index;
    _Py_SET_OPCODE(*instr, opcode_module);
    return 0;
}



/* Attribute specialization */

typedef enum {
    OVERRIDING, /* Is an overriding descriptor, and will remain so. */
    METHOD, /* Attribute has Py_TPFLAGS_METHOD_DESCRIPTOR set */
    PROPERTY, /* Is a property */
    OBJECT_SLOT, /* Is an object slot descriptor */
    OTHER_SLOT, /* Is a slot descriptor of another type */
    NON_OVERRIDING, /* Is another non-overriding descriptor, and is an instance of an immutable class*/
    BUILTIN_CLASSMETHOD, /* Builtin methods with METH_CLASS */
    PYTHON_CLASSMETHOD, /* Python classmethod(func) object */
    NON_DESCRIPTOR, /* Is not a descriptor, and is an instance of an immutable class */
    MUTABLE,   /* Instance of a mutable class; might, or might not, be a descriptor */
    ABSENT, /* Attribute is not present on the class */
    DUNDER_CLASS, /* __class__ attribute */
    GETSET_OVERRIDDEN /* __getattribute__ or __setattr__ has been overridden */
} DescriptorClassification;


static DescriptorClassification
analyze_descriptor(PyTypeObject *type, PyObject *name, PyObject **descr, int store)
{
    if (store) {
        if (type->tp_setattro != PyObject_GenericSetAttr) {
            *descr = NULL;
            return GETSET_OVERRIDDEN;
        }
    }
    else {
        if (type->tp_getattro != PyObject_GenericGetAttr) {
            *descr = NULL;
            return GETSET_OVERRIDDEN;
        }
    }
    PyObject *descriptor = _PyType_Lookup(type, name);
    *descr = descriptor;
    if (descriptor == NULL) {
        return ABSENT;
    }
    PyTypeObject *desc_cls = Py_TYPE(descriptor);
    if (!(desc_cls->tp_flags & Py_TPFLAGS_IMMUTABLETYPE)) {
        return MUTABLE;
    }
    if (desc_cls->tp_descr_set) {
        if (desc_cls == &PyMemberDescr_Type) {
            PyMemberDescrObject *member = (PyMemberDescrObject *)descriptor;
            struct PyMemberDef *dmem = member->d_member;
            if (dmem->type == T_OBJECT_EX) {
                return OBJECT_SLOT;
            }
            return OTHER_SLOT;
        }
        if (desc_cls == &PyProperty_Type) {
            return PROPERTY;
        }
        if (PyUnicode_CompareWithASCIIString(name, "__class__") == 0) {
            if (descriptor == _PyType_Lookup(&PyBaseObject_Type, name)) {
                return DUNDER_CLASS;
            }
        }
        return OVERRIDING;
    }
    if (desc_cls->tp_descr_get) {
        if (desc_cls->tp_flags & Py_TPFLAGS_METHOD_DESCRIPTOR) {
            return METHOD;
        }
        if (Py_IS_TYPE(descriptor, &PyClassMethodDescr_Type)) {
            return BUILTIN_CLASSMETHOD;
        }
        if (Py_IS_TYPE(descriptor, &PyClassMethod_Type)) {
            return PYTHON_CLASSMETHOD;
        }
        return NON_OVERRIDING;
    }
    return NON_DESCRIPTOR;
}

static int
specialize_dict_access(
    PyObject *owner, _Py_CODEUNIT *instr, PyTypeObject *type,
    DescriptorClassification kind, PyObject *name,
    int base_op, int values_op, int hint_op)
{
    assert(kind == NON_OVERRIDING || kind == NON_DESCRIPTOR || kind == ABSENT ||
        kind == BUILTIN_CLASSMETHOD || kind == PYTHON_CLASSMETHOD);
    // No descriptor, or non overriding.
    if ((type->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) {
        SPECIALIZATION_FAIL(base_op, SPEC_FAIL_ATTR_NOT_MANAGED_DICT);
        return 0;
    }
    _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
    PyObject **dictptr = _PyObject_ManagedDictPointer(owner);
    PyDictObject *dict = (PyDictObject *)*dictptr;
    if (dict == NULL) {
        // Virtual dictionary
        PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys;
        assert(PyUnicode_CheckExact(name));
        Py_ssize_t index = _PyDictKeys_StringLookup(keys, name);
        assert (index != DKIX_ERROR);
        if (index != (uint16_t)index) {
            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
            return 0;
        }
        write_u32(cache->version, type->tp_version_tag);
        cache->index = (uint16_t)index;
        _Py_SET_OPCODE(*instr, values_op);
    }
    else {
        if (!PyDict_CheckExact(dict)) {
            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_NO_DICT);
            return 0;
        }
        // We found an instance with a __dict__.
        PyObject *value = NULL;
        Py_ssize_t hint =
            _PyDict_GetItemHint(dict, name, -1, &value);
        if (hint != (uint16_t)hint) {
            SPECIALIZATION_FAIL(base_op, SPEC_FAIL_OUT_OF_RANGE);
            return 0;
        }
        cache->index = (uint16_t)hint;
        write_u32(cache->version, type->tp_version_tag);
        _Py_SET_OPCODE(*instr, hint_op);
    }
    return 1;
}

int
_Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
    assert(_PyOpcode_InlineCacheEntries[LOAD_ATTR] ==
           INLINE_CACHE_ENTRIES_LOAD_ATTR);
    _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
    if (PyModule_CheckExact(owner)) {
        int err = specialize_module_load_attr(owner, instr, name, LOAD_ATTR,
                                              LOAD_ATTR_MODULE);
        if (err) {
            goto fail;
        }
        goto success;
    }
    PyTypeObject *type = Py_TYPE(owner);
    if (type->tp_dict == NULL) {
        if (PyType_Ready(type) < 0) {
            return -1;
        }
    }
    PyObject *descr;
    DescriptorClassification kind = analyze_descriptor(type, name, &descr, 0);
    switch(kind) {
        case OVERRIDING:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
            goto fail;
        case METHOD:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_METHOD);
            goto fail;
        case PROPERTY:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_PROPERTY);
            goto fail;
        case OBJECT_SLOT:
        {
            PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
            struct PyMemberDef *dmem = member->d_member;
            Py_ssize_t offset = dmem->offset;
            if (dmem->flags & PY_AUDIT_READ) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_AUDITED_SLOT);
                goto fail;
            }
            if (offset != (uint16_t)offset) {
                SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OUT_OF_RANGE);
                goto fail;
            }
            assert(dmem->type == T_OBJECT_EX);
            assert(offset > 0);
            cache->index = (uint16_t)offset;
            write_u32(cache->version, type->tp_version_tag);
            _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT);
            goto success;
        }
        case DUNDER_CLASS:
        {
            Py_ssize_t offset = offsetof(PyObject, ob_type);
            assert(offset == (uint16_t)offset);
            cache->index = (uint16_t)offset;
            write_u32(cache->version, type->tp_version_tag);
            _Py_SET_OPCODE(*instr, LOAD_ATTR_SLOT);
            goto success;
        }
        case OTHER_SLOT:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
            goto fail;
        case MUTABLE:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
            goto fail;
        case GETSET_OVERRIDDEN:
            SPECIALIZATION_FAIL(LOAD_ATTR, SPEC_FAIL_OVERRIDDEN);
            goto fail;
        case BUILTIN_CLASSMETHOD:
        case PYTHON_CLASSMETHOD:
        case NON_OVERRIDING:
        case NON_DESCRIPTOR:
        case ABSENT:
            break;
    }
    int err = specialize_dict_access(
        owner, instr, type, kind, name,
        LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_ATTR_WITH_HINT
    );
    if (err < 0) {
        return -1;
    }
    if (err) {
        goto success;
    }
fail:
    STAT_INC(LOAD_ATTR, failure);
    assert(!PyErr_Occurred());
    cache->counter = ADAPTIVE_CACHE_BACKOFF;
    return 0;
success:
    STAT_INC(LOAD_ATTR, success);
    assert(!PyErr_Occurred());
    cache->counter = initial_counter_value();
    return 0;
}

int
_Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
    assert(_PyOpcode_InlineCacheEntries[STORE_ATTR] ==
           INLINE_CACHE_ENTRIES_STORE_ATTR);
    _PyAttrCache *cache = (_PyAttrCache *)(instr + 1);
    PyTypeObject *type = Py_TYPE(owner);
    if (PyModule_CheckExact(owner)) {
        SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
        goto fail;
    }
    PyObject *descr;
    DescriptorClassification kind = analyze_descriptor(type, name, &descr, 1);
    switch(kind) {
        case OVERRIDING:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_OVERRIDING_DESCRIPTOR);
            goto fail;
        case METHOD:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_METHOD);
            goto fail;
        case PROPERTY:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_PROPERTY);
            goto fail;
        case OBJECT_SLOT:
        {
            PyMemberDescrObject *member = (PyMemberDescrObject *)descr;
            struct PyMemberDef *dmem = member->d_member;
            Py_ssize_t offset = dmem->offset;
            if (dmem->flags & READONLY) {
                SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_READ_ONLY);
                goto fail;
            }
            if (offset != (uint16_t)offset) {
                SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OUT_OF_RANGE);
                goto fail;
            }
            assert(dmem->type == T_OBJECT_EX);
            assert(offset > 0);
            cache->index = (uint16_t)offset;
            write_u32(cache->version, type->tp_version_tag);
            _Py_SET_OPCODE(*instr, STORE_ATTR_SLOT);
            goto success;
        }
        case DUNDER_CLASS:
        case OTHER_SLOT:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_NON_OBJECT_SLOT);
            goto fail;
        case MUTABLE:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_ATTR_MUTABLE_CLASS);
            goto fail;
        case GETSET_OVERRIDDEN:
            SPECIALIZATION_FAIL(STORE_ATTR, SPEC_FAIL_OVERRIDDEN);
            goto fail;
        case BUILTIN_CLASSMETHOD:
        case PYTHON_CLASSMETHOD:
        case NON_OVERRIDING:
        case NON_DESCRIPTOR:
        case ABSENT:
            break;
    }

    int err = specialize_dict_access(
        owner, instr, type, kind, name,
        STORE_ATTR, STORE_ATTR_INSTANCE_VALUE, STORE_ATTR_WITH_HINT
    );
    if (err < 0) {
        return -1;
    }
    if (err) {
        goto success;
    }
fail:
    STAT_INC(STORE_ATTR, failure);
    assert(!PyErr_Occurred());
    cache->counter = ADAPTIVE_CACHE_BACKOFF;
    return 0;
success:
    STAT_INC(STORE_ATTR, success);
    assert(!PyErr_Occurred());
    cache->counter = initial_counter_value();
    return 0;
}


#ifdef Py_STATS
static int
load_method_fail_kind(DescriptorClassification kind)
{
    switch (kind) {
        case OVERRIDING:
            return SPEC_FAIL_LOAD_METHOD_OVERRIDING_DESCRIPTOR;
        case METHOD:
            return SPEC_FAIL_LOAD_METHOD_METHOD;
        case PROPERTY:
            return SPEC_FAIL_LOAD_METHOD_PROPERTY;
        case OBJECT_SLOT:
            return SPEC_FAIL_LOAD_METHOD_OBJECT_SLOT;
        case OTHER_SLOT:
            return SPEC_FAIL_LOAD_METHOD_NON_OBJECT_SLOT;
        case DUNDER_CLASS:
            return SPEC_FAIL_OTHER;
        case MUTABLE:
            return SPEC_FAIL_LOAD_METHOD_MUTABLE_CLASS;
        case GETSET_OVERRIDDEN:
            return SPEC_FAIL_OVERRIDDEN;
        case BUILTIN_CLASSMETHOD:
            return SPEC_FAIL_LOAD_METHOD_BUILTIN_CLASS_METHOD;
        case PYTHON_CLASSMETHOD:
            return SPEC_FAIL_LOAD_METHOD_CLASS_METHOD_OBJ;
        case NON_OVERRIDING:
            return SPEC_FAIL_LOAD_METHOD_NON_OVERRIDING_DESCRIPTOR;
        case NON_DESCRIPTOR:
            return SPEC_FAIL_LOAD_METHOD_NOT_DESCRIPTOR;
        case ABSENT:
            return SPEC_FAIL_LOAD_METHOD_INSTANCE_ATTRIBUTE;
    }
    Py_UNREACHABLE();
}
#endif

static int
specialize_class_load_method(PyObject *owner, _Py_CODEUNIT *instr,
                             PyObject *name)
{
    _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
    PyObject *descr = NULL;
    DescriptorClassification kind = 0;
    kind = analyze_descriptor((PyTypeObject *)owner, name, &descr, 0);
    switch (kind) {
        case METHOD:
        case NON_DESCRIPTOR:
            write_u32(cache->type_version, ((PyTypeObject *)owner)->tp_version_tag);
            write_obj(cache->descr, descr);
            _Py_SET_OPCODE(*instr, LOAD_METHOD_CLASS);
            return 0;
#ifdef Py_STATS
        case ABSENT:
            if (_PyType_Lookup(Py_TYPE(owner), name) != NULL) {
                SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_LOAD_METHOD_METACLASS_ATTRIBUTE);
            }
            else {
                SPECIALIZATION_FAIL(LOAD_METHOD, SPEC_FAIL_EXPECTED_ERROR);
            }
            return -1;
#endif
        default:
            SPECIALIZATION_FAIL(LOAD_METHOD, load_method_fail_kind(kind));
            return -1;
    }
}

typedef enum {
    MANAGED_VALUES = 1,
    MANAGED_DICT = 2,
    OFFSET_DICT = 3,
    NO_DICT = 4
} ObjectDictKind;

// Please collect stats carefully before and after modifying. A subtle change
// can cause a significant drop in cache hits. A possible test is
// python.exe -m test_typing test_re test_dis test_zlib.
int
_Py_Specialize_LoadMethod(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name)
{
    assert(_PyOpcode_InlineCacheEntries[LOAD_METHOD] ==
           INLINE_CACHE_ENTRIES_LOAD_METHOD);
    _PyLoadMethodCache *cache = (_PyLoadMethodCache *)(instr + 1);
    PyTypeObject *owner_cls = Py_TYPE(owner);

    if (PyModule_CheckExact(owner)) {
        assert(INLINE_CACHE_ENTRIES_LOAD_ATTR <=
               INLINE_CACHE_ENTRIES_LOAD_METHOD);
        int err = specialize_module_load_attr(owner, instr, name, LOAD_METHOD,
                                              LOAD_METHOD_MODULE);
        if (err) {
            goto fail;
        }
        goto success;
    }
    if (owner_cls->tp_dict == NULL) {
        if (PyType_Ready(owner_cls) < 0) {
            return -1;
        }
    }
    if (PyType_Check(owner)) {
        int err = specialize_class_load_method(owner, instr, name);
        if (err) {
            goto fail;
        }
        goto success;
    }

    PyObject *descr = NULL;
    DescriptorClassification kind = 0;
    kind = analyze_descriptor(owner_cls, name, &descr, 0);
    assert(descr != NULL || kind == ABSENT || kind == GETSET_OVERRIDDEN);
    if (kind != METHOD) {