Code.py

# cython: language_level=3str
# cython: auto_pickle=False
#
#   Code output module
#

from __future__ import absolute_import

import cython
cython.declare(os=object, re=object, operator=object, textwrap=object,
               Template=object, Naming=object, Options=object, StringEncoding=object,
               Utils=object, SourceDescriptor=object, StringIOTree=object,
               DebugFlags=object, basestring=object, defaultdict=object,
               closing=object, partial=object)

import hashlib
import operator
import os
import re
import shutil
import textwrap
from string import Template
from functools import partial
from contextlib import closing, contextmanager
from collections import defaultdict

from . import Naming
from . import Options
from . import DebugFlags
from . import StringEncoding
from .. import Utils
from .Scanning import SourceDescriptor
from ..StringIOTree import StringIOTree

try:
    from __builtin__ import basestring
except ImportError:
    from builtins import str as basestring


non_portable_builtins_map = {
    # builtins that have different names in different Python versions
    'bytes'         : ('PY_MAJOR_VERSION < 3',  'str'),
    'unicode'       : ('PY_MAJOR_VERSION >= 3', 'str'),
    'basestring'    : ('PY_MAJOR_VERSION >= 3', 'str'),
    'xrange'        : ('PY_MAJOR_VERSION >= 3', 'range'),
    'raw_input'     : ('PY_MAJOR_VERSION >= 3', 'input'),
}

ctypedef_builtins_map = {
    # types of builtins in "ctypedef class" statements which we don't
    # import either because the names conflict with C types or because
    # the type simply is not exposed.
    'py_int'             : '&PyInt_Type',
    'py_long'            : '&PyLong_Type',
    'py_float'           : '&PyFloat_Type',
    'wrapper_descriptor' : '&PyWrapperDescr_Type',
}

basicsize_builtins_map = {
    # builtins whose type has a different tp_basicsize than sizeof(...)
    'PyTypeObject': 'PyHeapTypeObject',
}

uncachable_builtins = [
    # Global/builtin names that cannot be cached because they may or may not
    # be available at import time, for various reasons:
    ## - Py3.7+
    'breakpoint',  # might deserve an implementation in Cython
    ## - Py3.4+
    '__loader__',
    '__spec__',
    ## - Py3+
    'BlockingIOError',
    'BrokenPipeError',
    'ChildProcessError',
    'ConnectionAbortedError',
    'ConnectionError',
    'ConnectionRefusedError',
    'ConnectionResetError',
    'FileExistsError',
    'FileNotFoundError',
    'InterruptedError',
    'IsADirectoryError',
    'ModuleNotFoundError',
    'NotADirectoryError',
    'PermissionError',
    'ProcessLookupError',
    'RecursionError',
    'ResourceWarning',
    #'StopAsyncIteration',  # backported
    'TimeoutError',
    '__build_class__',
    'ascii',  # might deserve an implementation in Cython
    #'exec',  # implemented in Cython
    ## - platform specific
    'WindowsError',
    ## - others
    '_',  # e.g. used by gettext
]

special_py_methods = cython.declare(frozenset, frozenset((
    '__cinit__', '__dealloc__', '__richcmp__', '__next__',
    '__await__', '__aiter__', '__anext__',
    '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__',
    '__getcharbuffer__', '__getbuffer__', '__releasebuffer__',
)))

modifier_output_mapper = {
    'inline': 'CYTHON_INLINE'
}.get


class IncludeCode(object):
    """
    An include file and/or verbatim C code to be included in the
    generated sources.
    """
    # attributes:
    #
    #  pieces    {order: unicode}: pieces of C code to be generated.
    #            For the included file, the key "order" is zero.
    #            For verbatim include code, the "order" is the "order"
    #            attribute of the original IncludeCode where this piece
    #            of C code was first added. This is needed to prevent
    #            duplication if the same include code is found through
    #            multiple cimports.
    #  location  int: where to put this include in the C sources, one
    #            of the constants INITIAL, EARLY, LATE
    #  order     int: sorting order (automatically set by increasing counter)

    # Constants for location. If the same include occurs with different
    # locations, the earliest one takes precedense.
    INITIAL = 0
    EARLY = 1
    LATE = 2

    counter = 1   # Counter for "order"

    def __init__(self, include=None, verbatim=None, late=True, initial=False):
        self.order = self.counter
        type(self).counter += 1
        self.pieces = {}

        if include:
            if include[0] == '<' and include[-1] == '>':
                self.pieces[0] = u'#include {0}'.format(include)
                late = False  # system include is never late
            else:
                self.pieces[0] = u'#include "{0}"'.format(include)

        if verbatim:
            self.pieces[self.order] = verbatim

        if initial:
            self.location = self.INITIAL
        elif late:
            self.location = self.LATE
        else:
            self.location = self.EARLY

    def dict_update(self, d, key):
        """
        Insert `self` in dict `d` with key `key`. If that key already
        exists, update the attributes of the existing value with `self`.
        """
        if key in d:
            other = d[key]
            other.location = min(self.location, other.location)
            other.pieces.update(self.pieces)
        else:
            d[key] = self

    def sortkey(self):
        return self.order

    def mainpiece(self):
        """
        Return the main piece of C code, corresponding to the include
        file. If there was no include file, return None.
        """
        return self.pieces.get(0)

    def write(self, code):
        # Write values of self.pieces dict, sorted by the keys
        for k in sorted(self.pieces):
            code.putln(self.pieces[k])


def get_utility_dir():
    # make this a function and not global variables:
    # http://trac.cython.org/cython_trac/ticket/475
    Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    return os.path.join(Cython_dir, "Utility")

read_utilities_hook = None
"""
Override the hook for reading a utilities file that contains code fragments used
by the codegen.

The hook functions takes the path of the utilities file, and returns a list
of strings, one per line.

The default behavior is to open a file relative to get_utility_dir().
"""

def read_utilities_from_utility_dir(path):
    """
    Read all lines of the file at the provided path from a path relative
    to get_utility_dir().
    """
    filename = os.path.join(get_utility_dir(), path)
    with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f:
        return f.readlines()

# by default, read utilities from the utility directory.
read_utilities_hook = read_utilities_from_utility_dir

class UtilityCodeBase(object):
    """
    Support for loading utility code from a file.

    Code sections in the file can be specified as follows:

        ##### MyUtility.proto #####

        [proto declarations]

        ##### MyUtility.init #####

        [code run at module initialization]

        ##### MyUtility #####
        #@requires: MyOtherUtility
        #@substitute: naming

        [definitions]

        ##### MyUtility #####
        #@substitute: tempita

        [requires tempita substitution
         - context can't be specified here though so only
           tempita utility that requires no external context
           will benefit from this tag
         - only necessary when @required from non-tempita code]

    for prototypes and implementation respectively.  For non-python or
    -cython files backslashes should be used instead.  5 to 30 comment
    characters may be used on either side.

    If the @cname decorator is not used and this is a CythonUtilityCode,
    one should pass in the 'name' keyword argument to be used for name
    mangling of such entries.
    """

    is_cython_utility = False
    _utility_cache = {}

    @classmethod
    def _add_utility(cls, utility, type, lines, begin_lineno, tags=None):
        if utility is None:
            return

        code = '\n'.join(lines)
        if tags and 'substitute' in tags and 'naming' in tags['substitute']:
            try:
                code = Template(code).substitute(vars(Naming))
            except (KeyError, ValueError) as e:
                raise RuntimeError("Error parsing templated utility code of type '%s' at line %d: %s" % (
                    type, begin_lineno, e))

        # remember correct line numbers at least until after templating
        code = '\n' * begin_lineno + code

        if type == 'proto':
            utility[0] = code
        elif type == 'impl':
            utility[1] = code
        else:
            all_tags = utility[2]
            all_tags[type] = code

        if tags:
            all_tags = utility[2]
            for name, values in tags.items():
                all_tags.setdefault(name, set()).update(values)

    @classmethod
    def load_utilities_from_file(cls, path):
        utilities = cls._utility_cache.get(path)
        if utilities:
            return utilities

        _, ext = os.path.splitext(path)
        if ext in ('.pyx', '.py', '.pxd', '.pxi'):
            comment = '#'
            strip_comments = partial(re.compile(r'^\s*#(?!\s*cython\s*:).*').sub, '')
            rstrip = StringEncoding._unicode.rstrip
        else:
            comment = '/'
            strip_comments = partial(re.compile(r'^\s*//.*|/\*[^*]*\*/').sub, '')
            rstrip = partial(re.compile(r'\s+(\\?)$').sub, r'\1')
        match_special = re.compile(
            (r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|'
             r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)') %
            {'C': comment}).match
        match_type = re.compile(r'(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match

        all_lines = read_utilities_hook(path)

        utilities = defaultdict(lambda: [None, None, {}])
        lines = []
        tags = defaultdict(set)
        utility = type = None
        begin_lineno = 0

        for lineno, line in enumerate(all_lines):
            m = match_special(line)
            if m:
                if m.group('name'):
                    cls._add_utility(utility, type, lines, begin_lineno, tags)

                    begin_lineno = lineno + 1
                    del lines[:]
                    tags.clear()

                    name = m.group('name')
                    mtype = match_type(name)
                    if mtype:
                        name, type = mtype.groups()
                    else:
                        type = 'impl'
                    utility = utilities[name]
                else:
                    tags[m.group('tag')].add(m.group('value'))
                    lines.append('')  # keep line number correct
            else:
                lines.append(rstrip(strip_comments(line)))

        if utility is None:
            raise ValueError("Empty utility code file")

        # Don't forget to add the last utility code
        cls._add_utility(utility, type, lines, begin_lineno, tags)

        utilities = dict(utilities)  # un-defaultdict-ify
        cls._utility_cache[path] = utilities
        return utilities

    @classmethod
    def load(cls, util_code_name, from_file, **kwargs):
        """
        Load utility code from a file specified by from_file (relative to
        Cython/Utility) and name util_code_name.
        """

        if '::' in util_code_name:
            from_file, util_code_name = util_code_name.rsplit('::', 1)
        assert from_file
        utilities = cls.load_utilities_from_file(from_file)
        proto, impl, tags = utilities[util_code_name]

        if tags:
            if "substitute" in tags and "tempita" in tags["substitute"]:
                if not issubclass(cls, TempitaUtilityCode):
                    return TempitaUtilityCode.load(util_code_name, from_file, **kwargs)
            orig_kwargs = kwargs.copy()
            for name, values in tags.items():
                if name in kwargs:
                    continue
                # only pass lists when we have to: most argument expect one value or None
                if name == 'requires':
                    if orig_kwargs:
                        values = [cls.load(dep, from_file, **orig_kwargs)
                                  for dep in sorted(values)]
                    else:
                        # dependencies are rarely unique, so use load_cached() when we can
                        values = [cls.load_cached(dep, from_file)
                                  for dep in sorted(values)]
                elif name == 'substitute':
                    # don't want to pass "naming" or "tempita" to the constructor
                    # since these will have been handled
                    values = values - {'naming', 'tempita'}
                    if not values:
                        continue
                elif not values:
                    values = None
                elif len(values) == 1:
                    values = list(values)[0]
                kwargs[name] = values

        if proto is not None:
            kwargs['proto'] = proto
        if impl is not None:
            kwargs['impl'] = impl

        if 'name' not in kwargs:
            kwargs['name'] = util_code_name

        if 'file' not in kwargs and from_file:
            kwargs['file'] = from_file
        return cls(**kwargs)

    @classmethod
    def load_cached(cls, utility_code_name, from_file, __cache={}):
        """
        Calls .load(), but using a per-type cache based on utility name and file name.
        """
        key = (utility_code_name, from_file, cls)
        try:
            return __cache[key]
        except KeyError:
            pass
        code = __cache[key] = cls.load(utility_code_name, from_file)
        return code

    @classmethod
    def load_as_string(cls, util_code_name, from_file, **kwargs):
        """
        Load a utility code as a string. Returns (proto, implementation)
        """
        util = cls.load(util_code_name, from_file, **kwargs)
        proto, impl = util.proto, util.impl
        return util.format_code(proto), util.format_code(impl)

    def format_code(self, code_string, replace_empty_lines=re.compile(r'\n\n+').sub):
        """
        Format a code section for output.
        """
        if code_string:
            code_string = replace_empty_lines('\n', code_string.strip()) + '\n\n'
        return code_string

    def __str__(self):
        return "<%s(%s)>" % (type(self).__name__, self.name)

    def get_tree(self, **kwargs):
        return None

    def __deepcopy__(self, memodict=None):
        # No need to deep-copy utility code since it's essentially immutable.
        return self


class UtilityCode(UtilityCodeBase):
    """
    Stores utility code to add during code generation.

    See GlobalState.put_utility_code.

    hashes/equals by instance

    proto           C prototypes
    impl            implementation code
    init            code to call on module initialization
    requires        utility code dependencies
    proto_block     the place in the resulting file where the prototype should
                    end up
    name            name of the utility code (or None)
    file            filename of the utility code file this utility was loaded
                    from (or None)
    """

    def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
                 proto_block='utility_code_proto', name=None, file=None):
        # proto_block: Which code block to dump prototype in. See GlobalState.
        self.proto = proto
        self.impl = impl
        self.init = init
        self.cleanup = cleanup
        self.requires = requires
        self._cache = {}
        self.specialize_list = []
        self.proto_block = proto_block
        self.name = name
        self.file = file

    def __hash__(self):
        return hash((self.proto, self.impl))

    def __eq__(self, other):
        if self is other:
            return True
        self_type, other_type = type(self), type(other)
        if self_type is not other_type and not (isinstance(other, self_type) or isinstance(self, other_type)):
            return False

        self_proto = getattr(self, 'proto', None)
        other_proto = getattr(other, 'proto', None)
        return (self_proto, self.impl) == (other_proto, other.impl)

    def none_or_sub(self, s, context):
        """
        Format a string in this utility code with context. If None, do nothing.
        """
        if s is None:
            return None
        return s % context

    def specialize(self, pyrex_type=None, **data):
        name = self.name
        if pyrex_type is not None:
            data['type'] = pyrex_type.empty_declaration_code()
            data['type_name'] = pyrex_type.specialization_name()
            name = "%s[%s]" % (name, data['type_name'])
        # Dicts aren't hashable...
        key = tuple(sorted(data.items()))
        try:
            return self._cache[key]
        except KeyError:
            if self.requires is None:
                requires = None
            else:
                requires = [r.specialize(data) for r in self.requires]

            s = self._cache[key] = UtilityCode(
                self.none_or_sub(self.proto, data),
                self.none_or_sub(self.impl, data),
                self.none_or_sub(self.init, data),
                self.none_or_sub(self.cleanup, data),
                requires,
                self.proto_block,
                name,
            )

            self.specialize_list.append(s)
            return s

    def inject_string_constants(self, impl, output):
        """Replace 'PYIDENT("xyz")' by a constant Python identifier cname.
        """
        if 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl:
            return False, impl

        replacements = {}
        def externalise(matchobj):
            key = matchobj.groups()
            try:
                cname = replacements[key]
            except KeyError:
                str_type, name = key
                cname = replacements[key] = output.get_py_string_const(
                        StringEncoding.EncodedString(name), identifier=str_type == 'IDENT').cname
            return cname

        impl = re.sub(r'PY(IDENT|UNICODE)\("([^"]+)"\)', externalise, impl)
        assert 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl
        return True, impl

    def inject_unbound_methods(self, impl, output):
        """Replace 'UNBOUND_METHOD(type, "name")' by a constant Python identifier cname.
        """
        if 'CALL_UNBOUND_METHOD(' not in impl:
            return False, impl

        def externalise(matchobj):
            type_cname, method_name, obj_cname, args = matchobj.groups()
            args = [arg.strip() for arg in args[1:].split(',')] if args else []
            assert len(args) < 3, "CALL_UNBOUND_METHOD() does not support %d call arguments" % len(args)
            return output.cached_unbound_method_call_code(obj_cname, type_cname, method_name, args)

        impl = re.sub(
            r'CALL_UNBOUND_METHOD\('
            r'([a-zA-Z_]+),'      # type cname
            r'\s*"([^"]+)",'      # method name
            r'\s*([^),]+)'        # object cname
            r'((?:,[^),]+)*)'     # args*
            r'\)', externalise, impl)
        assert 'CALL_UNBOUND_METHOD(' not in impl

        return True, impl

    def wrap_c_strings(self, impl):
        """Replace CSTRING('''xyz''') by a C compatible string
        """
        if 'CSTRING(' not in impl:
            return impl

        def split_string(matchobj):
            content = matchobj.group(1).replace('"', '\042')
            return ''.join(
                '"%s\\n"\n' % line if not line.endswith('\\') or line.endswith('\\\\') else '"%s"\n' % line[:-1]
                for line in content.splitlines())

        impl = re.sub(r'CSTRING\(\s*"""([^"]*(?:"[^"]+)*)"""\s*\)', split_string, impl)
        assert 'CSTRING(' not in impl
        return impl

    def put_code(self, output):
        if self.requires:
            for dependency in self.requires:
                output.use_utility_code(dependency)
        if self.proto:
            writer = output[self.proto_block]
            writer.putln("/* %s.proto */" % self.name)
            writer.put_or_include(
                self.format_code(self.proto), '%s_proto' % self.name)
        if self.impl:
            impl = self.format_code(self.wrap_c_strings(self.impl))
            is_specialised1, impl = self.inject_string_constants(impl, output)
            is_specialised2, impl = self.inject_unbound_methods(impl, output)
            writer = output['utility_code_def']
            writer.putln("/* %s */" % self.name)
            if not (is_specialised1 or is_specialised2):
                # no module specific adaptations => can be reused
                writer.put_or_include(impl, '%s_impl' % self.name)
            else:
                writer.put(impl)
        if self.init:
            writer = output['init_globals']
            writer.putln("/* %s.init */" % self.name)
            if isinstance(self.init, basestring):
                writer.put(self.format_code(self.init))
            else:
                self.init(writer, output.module_pos)
            # 'init' code can end with an 'if' statement for an error condition like:
            # if (check_ok()) ; else
            writer.putln(writer.error_goto_if_PyErr(output.module_pos))
            writer.putln()
        if self.cleanup and Options.generate_cleanup_code:
            writer = output['cleanup_globals']
            writer.putln("/* %s.cleanup */" % self.name)
            if isinstance(self.cleanup, basestring):
                writer.put_or_include(
                    self.format_code(self.cleanup),
                    '%s_cleanup' % self.name)
            else:
                self.cleanup(writer, output.module_pos)


def sub_tempita(s, context, file=None, name=None):
    "Run tempita on string s with given context."
    if not s:
        return None

    if file:
        context['__name'] = "%s:%s" % (file, name)
    elif name:
        context['__name'] = name

    from ..Tempita import sub
    return sub(s, **context)


class TempitaUtilityCode(UtilityCode):
    def __init__(self, name=None, proto=None, impl=None, init=None, file=None, context=None, **kwargs):
        if context is None:
            context = {}
        proto = sub_tempita(proto, context, file, name)
        impl = sub_tempita(impl, context, file, name)
        init = sub_tempita(init, context, file, name)
        super(TempitaUtilityCode, self).__init__(
            proto, impl, init=init, name=name, file=file, **kwargs)

    @classmethod
    def load_cached(cls, utility_code_name, from_file=None, context=None, __cache={}):
        context_key = tuple(sorted(context.items())) if context else None
        assert hash(context_key) is not None  # raise TypeError if not hashable
        key = (cls, from_file, utility_code_name, context_key)
        try:
            return __cache[key]
        except KeyError:
            pass
        code = __cache[key] = cls.load(utility_code_name, from_file, context=context)
        return code

    def none_or_sub(self, s, context):
        """
        Format a string in this utility code with context. If None, do nothing.
        """
        if s is None:
            return None
        return sub_tempita(s, context, self.file, self.name)


class LazyUtilityCode(UtilityCodeBase):
    """
    Utility code that calls a callback with the root code writer when
    available. Useful when you only have 'env' but not 'code'.
    """
    __name__ = '<lazy>'
    requires = None

    def __init__(self, callback):
        self.callback = callback

    def put_code(self, globalstate):
        utility = self.callback(globalstate.rootwriter)
        globalstate.use_utility_code(utility)


class FunctionState(object):
    # return_label     string          function return point label
    # error_label      string          error catch point label
    # error_without_exception  boolean Can go to the error label without an exception (e.g. __next__ can return NULL)
    # continue_label   string          loop continue point label
    # break_label      string          loop break point label
    # return_from_error_cleanup_label string
    # label_counter    integer         counter for naming labels
    # in_try_finally   boolean         inside try of try...finally
    # exc_vars         (string * 3)    exception variables for reraise, or None
    # can_trace        boolean         line tracing is supported in the current context
    # scope            Scope           the scope object of the current function

    # Not used for now, perhaps later
    def __init__(self, owner, names_taken=set(), scope=None):
        self.names_taken = names_taken
        self.owner = owner
        self.scope = scope

        self.error_label = None
        self.label_counter = 0
        self.labels_used = set()
        self.return_label = self.new_label()
        self.new_error_label()
        self.continue_label = None
        self.break_label = None
        self.yield_labels = []

        self.in_try_finally = 0
        self.exc_vars = None
        self.current_except = None
        self.can_trace = False
        self.gil_owned = True

        self.temps_allocated = []  # of (name, type, manage_ref, static)
        self.temps_free = {}  # (type, manage_ref) -> list of free vars with same type/managed status
        self.temps_used_type = {}  # name -> (type, manage_ref)
        self.zombie_temps = set()  # temps that must not be reused after release
        self.temp_counter = 0
        self.closure_temps = None

        # This is used to collect temporaries, useful to find out which temps
        # need to be privatized in parallel sections
        self.collect_temps_stack = []

        # This is used for the error indicator, which needs to be local to the
        # function. It used to be global, which relies on the GIL being held.
        # However, exceptions may need to be propagated through 'nogil'
        # sections, in which case we introduce a race condition.
        self.should_declare_error_indicator = False
        self.uses_error_indicator = False

        self.error_without_exception = False

    # safety checks

    def validate_exit(self):
        # validate that all allocated temps have been freed
        if self.temps_allocated:
            leftovers = self.temps_in_use()
            if leftovers:
                msg = "TEMPGUARD: Temps left over at end of '%s': %s" % (self.scope.name, ', '.join([
                    '%s [%s]' % (name, ctype)
                    for name, ctype, is_pytemp in sorted(leftovers)]),
                )
                #print(msg)
                raise RuntimeError(msg)

    # labels

    def new_label(self, name=None):
        n = self.label_counter
        self.label_counter = n + 1
        label = "%s%d" % (Naming.label_prefix, n)
        if name is not None:
            label += '_' + name
        return label

    def new_yield_label(self, expr_type='yield'):
        label = self.new_label('resume_from_%s' % expr_type)
        num_and_label = (len(self.yield_labels) + 1, label)
        self.yield_labels.append(num_and_label)
        return num_and_label

    def new_error_label(self, prefix=""):
        old_err_lbl = self.error_label
        self.error_label = self.new_label(prefix + 'error')
        return old_err_lbl

    def get_loop_labels(self):
        return (
            self.continue_label,
            self.break_label)

    def set_loop_labels(self, labels):
        (self.continue_label,
         self.break_label) = labels

    def new_loop_labels(self, prefix=""):
        old_labels = self.get_loop_labels()
        self.set_loop_labels(
            (self.new_label(prefix + "continue"),
             self.new_label(prefix + "break")))
        return old_labels

    def get_all_labels(self):
        return (
            self.continue_label,
            self.break_label,
            self.return_label,
            self.error_label)

    def set_all_labels(self, labels):
        (self.continue_label,
         self.break_label,
         self.return_label,
         self.error_label) = labels

    def all_new_labels(self):
        old_labels = self.get_all_labels()
        new_labels = []
        for old_label, name in zip(old_labels, ['continue', 'break', 'return', 'error']):
            if old_label:
                new_labels.append(self.new_label(name))
            else:
                new_labels.append(old_label)
        self.set_all_labels(new_labels)
        return old_labels

    def use_label(self, lbl):
        self.labels_used.add(lbl)

    def label_used(self, lbl):
        return lbl in self.labels_used

    # temp handling

    def allocate_temp(self, type, manage_ref, static=False, reusable=True):
        """
        Allocates a temporary (which may create a new one or get a previously
        allocated and released one of the same type). Type is simply registered
        and handed back, but will usually be a PyrexType.

        If type.needs_refcounting, manage_ref comes into play. If manage_ref is set to
        True, the temp will be decref-ed on return statements and in exception
        handling clauses. Otherwise the caller has to deal with any reference
        counting of the variable.

        If not type.needs_refcounting, then manage_ref will be ignored, but it
        still has to be passed. It is recommended to pass False by convention
        if it is known that type will never be a reference counted type.

        static=True marks the temporary declaration with "static".
        This is only used when allocating backing store for a module-level
        C array literals.

        if reusable=False, the temp will not be reused after release.

        A C string referring to the variable is returned.
        """
        if type.is_cv_qualified and not type.is_reference:
            type = type.cv_base_type
        elif type.is_reference and not type.is_fake_reference:
            type = type.ref_base_type
        elif type.is_cfunction:
            from . import PyrexTypes
            type = PyrexTypes.c_ptr_type(type)  # A function itself isn't an l-value
        elif type.is_cpp_class and not type.is_fake_reference and self.scope.directives['cpp_locals']:
            self.scope.use_utility_code(UtilityCode.load_cached("OptionalLocals", "CppSupport.cpp"))
        if not type.needs_refcounting:
            # Make manage_ref canonical, so that manage_ref will always mean
            # a decref is needed.
            manage_ref = False

        freelist = self.temps_free.get((type, manage_ref))
        if reusable and freelist is not None and freelist[0]:
            result = freelist[0].pop()
            freelist[1].remove(result)
        else:
            while True:
                self.temp_counter += 1
                result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter)
                if result not in self.names_taken: break
            self.temps_allocated.append((result, type, manage_ref, static))
            if not reusable:
                self.zombie_temps.add(result)
        self.temps_used_type[result] = (type, manage_ref)
        if DebugFlags.debug_temp_code_comments:
            self.owner.putln("/* %s allocated (%s)%s */" % (result, type, "" if reusable else " - zombie"))

        if self.collect_temps_stack:
            self.collect_temps_stack[-1].add((result, type))

        return result

    def release_temp(self, name):
        """
        Releases a temporary so that it can be reused by other code needing
        a temp of the same type.
        """
        type, manage_ref = self.temps_used_type[name]
        freelist = self.temps_free.get((type, manage_ref))
        if freelist is None:
            freelist = ([], set())  # keep order in list and make lookups in set fast
            self.temps_free[(type, manage_ref)] = freelist
        if name in freelist[1]:
            raise RuntimeError("Temp %s freed twice!" % name)
        if name not in self.zombie_temps:
            freelist[0].append(name)
        freelist[1].add(name)
        if DebugFlags.debug_temp_code_comments:
            self.owner.putln("/* %s released %s*/" % (
                name, " - zombie" if name in self.zombie_temps else ""))

    def temps_in_use(self):
        """Return a list of (cname,type,manage_ref) tuples of temp names and their type
        that are currently in use.
        """
        used = []
        for name, type, manage_ref, static in self.temps_allocated:
            freelist = self.temps_free.get((type, manage_ref))
            if freelist is None or name not in freelist[1]:
                used.append((name, type, manage_ref and type.needs_refcounting))
        return used

    def temps_holding_reference(self):
        """Return a list of (cname,type) tuples of temp names and their type
        that are currently in use. This includes only temps
        with a reference counted type which owns its reference.
        """
        return [(name, type)
                for name, type, manage_ref in self.temps_in_use()
                if manage_ref and type.needs_refcounting]

    def all_managed_temps(self):
        """Return a list of (cname, type) tuples of refcount-managed Python objects.
        """
        return [(cname, type)
                for cname, type, manage_ref, static in self.temps_allocated
                if manage_ref]

    def all_free_managed_temps(self):
        """Return a list of (cname, type) tuples of refcount-managed Python
        objects that are not currently in use.  This is used by
        try-except and try-finally blocks to clean up temps in the
        error case.
        """
        return sorted([  # Enforce deterministic order.
            (cname, type)
            for (type, manage_ref), freelist in self.temps_free.items() if manage_ref
            for cname in freelist[0]
        ])

    def start_collecting_temps(self):
        """
        Useful to find out which temps were used in a code block
        """
        self.collect_temps_stack.append(set())

    def stop_collecting_temps(self):
        return self.collect_temps_stack.pop()

    def init_closure_temps(self, scope):
        self.closure_temps = ClosureTempAllocator(scope)


class NumConst(object):
    """Global info about a Python number constant held by GlobalState.

    cname       string
    value       string
    py_type     string     int, long, float
    value_code  string     evaluation code if different from value
    """

    def __init__(self, cname, value, py_type, value_code=None):
        self.cname = cname
        self.value = value
        self.py_type = py_type
        self.value_code = value_code or value


class PyObjectConst(object):
    """Global info about a generic constant held by GlobalState.
    """
    # cname       string
    # type        PyrexType

    def __init__(self, cname, type):
        self.cname = cname
        self.type = type


cython.declare(possible_unicode_identifier=object, possible_bytes_identifier=object,
               replace_identifier=object, find_alphanums=object)
possible_unicode_identifier = re.compile(br"(?![0-9])\w+$".decode('ascii'), re.U).match
possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match
replace_identifier = re.compile(r'[^a-zA-Z0-9_]+').sub
find_alphanums = re.compile('([a-zA-Z0-9]+)').findall

class StringConst(object):
    """Global info about a C string constant held by GlobalState.
    """
    # cname            string
    # text             EncodedString or BytesLiteral
    # py_strings       {(identifier, encoding) : PyStringConst}

    def __init__(self, cname, text, byte_string):