fileutils.c

#include "Python.h"
#include "pycore_fileutils.h"     // fileutils definitions
#include "pycore_runtime.h"       // _PyRuntime
#include "osdefs.h"               // SEP
#include <locale.h>

#ifdef MS_WINDOWS
#  include <malloc.h>
#  include <windows.h>
extern int winerror_to_errno(int);
#endif

#ifdef HAVE_LANGINFO_H
#include <langinfo.h>
#endif

#ifdef HAVE_SYS_IOCTL_H
#include <sys/ioctl.h>
#endif

#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION
#include <iconv.h>
#endif

#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif /* HAVE_FCNTL_H */

#ifdef O_CLOEXEC
/* Does open() support the O_CLOEXEC flag? Possible values:

   -1: unknown
    0: open() ignores O_CLOEXEC flag, ex: Linux kernel older than 2.6.23
    1: open() supports O_CLOEXEC flag, close-on-exec is set

   The flag is used by _Py_open(), _Py_open_noraise(), io.FileIO
   and os.open(). */
int _Py_open_cloexec_works = -1;
#endif

// The value must be the same in unicodeobject.c.
#define MAX_UNICODE 0x10ffff

// mbstowcs() and mbrtowc() errors
static const size_t DECODE_ERROR = ((size_t)-1);
static const size_t INCOMPLETE_CHARACTER = (size_t)-2;


static int
get_surrogateescape(_Py_error_handler errors, int *surrogateescape)
{
    switch (errors)
    {
    case _Py_ERROR_STRICT:
        *surrogateescape = 0;
        return 0;
    case _Py_ERROR_SURROGATEESCAPE:
        *surrogateescape = 1;
        return 0;
    default:
        return -1;
    }
}


PyObject *
_Py_device_encoding(int fd)
{
    int valid;
    Py_BEGIN_ALLOW_THREADS
    _Py_BEGIN_SUPPRESS_IPH
    valid = isatty(fd);
    _Py_END_SUPPRESS_IPH
    Py_END_ALLOW_THREADS
    if (!valid)
        Py_RETURN_NONE;

#if defined(MS_WINDOWS)
    UINT cp;
    if (fd == 0)
        cp = GetConsoleCP();
    else if (fd == 1 || fd == 2)
        cp = GetConsoleOutputCP();
    else
        cp = 0;
    /* GetConsoleCP() and GetConsoleOutputCP() return 0 if the application
       has no console */
    if (cp == 0) {
        Py_RETURN_NONE;
    }

    return PyUnicode_FromFormat("cp%u", (unsigned int)cp);
#else
    return _Py_GetLocaleEncodingObject();
#endif
}


static size_t
is_valid_wide_char(wchar_t ch)
{
#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION
    /* Oracle Solaris doesn't use Unicode code points as wchar_t encoding
       for non-Unicode locales, which makes values higher than MAX_UNICODE
       possibly valid. */
    return 1;
#endif
    if (Py_UNICODE_IS_SURROGATE(ch)) {
        // Reject lone surrogate characters
        return 0;
    }
    if (ch > MAX_UNICODE) {
        // bpo-35883: Reject characters outside [U+0000; U+10ffff] range.
        // The glibc mbstowcs() UTF-8 decoder does not respect the RFC 3629,
        // it creates characters outside the [U+0000; U+10ffff] range:
        // https://sourceware.org/bugzilla/show_bug.cgi?id=2373
        return 0;
    }
    return 1;
}


static size_t
_Py_mbstowcs(wchar_t *dest, const char *src, size_t n)
{
    size_t count = mbstowcs(dest, src, n);
    if (dest != NULL && count != DECODE_ERROR) {
        for (size_t i=0; i < count; i++) {
            wchar_t ch = dest[i];
            if (!is_valid_wide_char(ch)) {
                return DECODE_ERROR;
            }
        }
    }
    return count;
}


#ifdef HAVE_MBRTOWC
static size_t
_Py_mbrtowc(wchar_t *pwc, const char *str, size_t len, mbstate_t *pmbs)
{
    assert(pwc != NULL);
    size_t count = mbrtowc(pwc, str, len, pmbs);
    if (count != 0 && count != DECODE_ERROR && count != INCOMPLETE_CHARACTER) {
        if (!is_valid_wide_char(*pwc)) {
            return DECODE_ERROR;
        }
    }
    return count;
}
#endif


#if !defined(_Py_FORCE_UTF8_FS_ENCODING) && !defined(MS_WINDOWS)

#define USE_FORCE_ASCII

extern int _Py_normalize_encoding(const char *, char *, size_t);

/* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale
   and POSIX locale. nl_langinfo(CODESET) announces an alias of the
   ASCII encoding, whereas mbstowcs() and wcstombs() functions use the
   ISO-8859-1 encoding. The problem is that os.fsencode() and os.fsdecode() use
   locale.getpreferredencoding() codec. For example, if command line arguments
   are decoded by mbstowcs() and encoded back by os.fsencode(), we get a
   UnicodeEncodeError instead of retrieving the original byte string.

   The workaround is enabled if setlocale(LC_CTYPE, NULL) returns "C",
   nl_langinfo(CODESET) announces "ascii" (or an alias to ASCII), and at least
   one byte in range 0x80-0xff can be decoded from the locale encoding. The
   workaround is also enabled on error, for example if getting the locale
   failed.

   On HP-UX with the C locale or the POSIX locale, nl_langinfo(CODESET)
   announces "roman8" but mbstowcs() uses Latin1 in practice. Force also the
   ASCII encoding in this case.

   Values of force_ascii:

       1: the workaround is used: Py_EncodeLocale() uses
          encode_ascii_surrogateescape() and Py_DecodeLocale() uses
          decode_ascii()
       0: the workaround is not used: Py_EncodeLocale() uses wcstombs() and
          Py_DecodeLocale() uses mbstowcs()
      -1: unknown, need to call check_force_ascii() to get the value
*/
static int force_ascii = -1;

static int
check_force_ascii(void)
{
    char *loc = setlocale(LC_CTYPE, NULL);
    if (loc == NULL) {
        goto error;
    }
    if (strcmp(loc, "C") != 0 && strcmp(loc, "POSIX") != 0) {
        /* the LC_CTYPE locale is different than C and POSIX */
        return 0;
    }

#if defined(HAVE_LANGINFO_H) && defined(CODESET)
    const char *codeset = nl_langinfo(CODESET);
    if (!codeset || codeset[0] == '\0') {
        /* CODESET is not set or empty */
        goto error;
    }

    char encoding[20];   /* longest name: "iso_646.irv_1991\0" */
    if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding))) {
        goto error;
    }

#ifdef __hpux
    if (strcmp(encoding, "roman8") == 0) {
        unsigned char ch;
        wchar_t wch;
        size_t res;

        ch = (unsigned char)0xA7;
        res = _Py_mbstowcs(&wch, (char*)&ch, 1);
        if (res != DECODE_ERROR && wch == L'\xA7') {
            /* On HP-UX with C locale or the POSIX locale,
               nl_langinfo(CODESET) announces "roman8", whereas mbstowcs() uses
               Latin1 encoding in practice. Force ASCII in this case.

               Roman8 decodes 0xA7 to U+00CF. Latin1 decodes 0xA7 to U+00A7. */
            return 1;
        }
    }
#else
    const char* ascii_aliases[] = {
        "ascii",
        /* Aliases from Lib/encodings/aliases.py */
        "646",
        "ansi_x3.4_1968",
        "ansi_x3.4_1986",
        "ansi_x3_4_1968",
        "cp367",
        "csascii",
        "ibm367",
        "iso646_us",
        "iso_646.irv_1991",
        "iso_ir_6",
        "us",
        "us_ascii",
        NULL
    };

    int is_ascii = 0;
    for (const char **alias=ascii_aliases; *alias != NULL; alias++) {
        if (strcmp(encoding, *alias) == 0) {
            is_ascii = 1;
            break;
        }
    }
    if (!is_ascii) {
        /* nl_langinfo(CODESET) is not "ascii" or an alias of ASCII */
        return 0;
    }

    for (unsigned int i=0x80; i<=0xff; i++) {
        char ch[1];
        wchar_t wch[1];
        size_t res;

        unsigned uch = (unsigned char)i;
        ch[0] = (char)uch;
        res = _Py_mbstowcs(wch, ch, 1);
        if (res != DECODE_ERROR) {
            /* decoding a non-ASCII character from the locale encoding succeed:
               the locale encoding is not ASCII, force ASCII */
            return 1;
        }
    }
    /* None of the bytes in the range 0x80-0xff can be decoded from the locale
       encoding: the locale encoding is really ASCII */
#endif   /* !defined(__hpux) */
    return 0;
#else
    /* nl_langinfo(CODESET) is not available: always force ASCII */
    return 1;
#endif   /* defined(HAVE_LANGINFO_H) && defined(CODESET) */

error:
    /* if an error occurred, force the ASCII encoding */
    return 1;
}


int
_Py_GetForceASCII(void)
{
    if (force_ascii == -1) {
        force_ascii = check_force_ascii();
    }
    return force_ascii;
}


void
_Py_ResetForceASCII(void)
{
    force_ascii = -1;
}


static int
encode_ascii(const wchar_t *text, char **str,
             size_t *error_pos, const char **reason,
             int raw_malloc, _Py_error_handler errors)
{
    char *result = NULL, *out;
    size_t len, i;
    wchar_t ch;

    int surrogateescape;
    if (get_surrogateescape(errors, &surrogateescape) < 0) {
        return -3;
    }

    len = wcslen(text);

    /* +1 for NULL byte */
    if (raw_malloc) {
        result = PyMem_RawMalloc(len + 1);
    }
    else {
        result = PyMem_Malloc(len + 1);
    }
    if (result == NULL) {
        return -1;
    }

    out = result;
    for (i=0; i<len; i++) {
        ch = text[i];

        if (ch <= 0x7f) {
            /* ASCII character */
            *out++ = (char)ch;
        }
        else if (surrogateescape && 0xdc80 <= ch && ch <= 0xdcff) {
            /* UTF-8b surrogate */
            *out++ = (char)(ch - 0xdc00);
        }
        else {
            if (raw_malloc) {
                PyMem_RawFree(result);
            }
            else {
                PyMem_Free(result);
            }
            if (error_pos != NULL) {
                *error_pos = i;
            }
            if (reason) {
                *reason = "encoding error";
            }
            return -2;
        }
    }
    *out = '\0';
    *str = result;
    return 0;
}
#else
int
_Py_GetForceASCII(void)
{
    return 0;
}

void
_Py_ResetForceASCII(void)
{
    /* nothing to do */
}
#endif   /* !defined(_Py_FORCE_UTF8_FS_ENCODING) && !defined(MS_WINDOWS) */


#if !defined(HAVE_MBRTOWC) || defined(USE_FORCE_ASCII)
static int
decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen,
             const char **reason, _Py_error_handler errors)
{
    wchar_t *res;
    unsigned char *in;
    wchar_t *out;
    size_t argsize = strlen(arg) + 1;

    int surrogateescape;
    if (get_surrogateescape(errors, &surrogateescape) < 0) {
        return -3;
    }

    if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t)) {
        return -1;
    }
    res = PyMem_RawMalloc(argsize * sizeof(wchar_t));
    if (!res) {
        return -1;
    }

    out = res;
    for (in = (unsigned char*)arg; *in; in++) {
        unsigned char ch = *in;
        if (ch < 128) {
            *out++ = ch;
        }
        else {
            if (!surrogateescape) {
                PyMem_RawFree(res);
                if (wlen) {
                    *wlen = in - (unsigned char*)arg;
                }
                if (reason) {
                    *reason = "decoding error";
                }
                return -2;
            }
            *out++ = 0xdc00 + ch;
        }
    }
    *out = 0;

    if (wlen != NULL) {
        *wlen = out - res;
    }
    *wstr = res;
    return 0;
}
#endif   /* !HAVE_MBRTOWC */

static int
decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen,
                      const char **reason, _Py_error_handler errors)
{
    wchar_t *res;
    size_t argsize;
    size_t count;
#ifdef HAVE_MBRTOWC
    unsigned char *in;
    wchar_t *out;
    mbstate_t mbs;
#endif

    int surrogateescape;
    if (get_surrogateescape(errors, &surrogateescape) < 0) {
        return -3;
    }

#ifdef HAVE_BROKEN_MBSTOWCS
    /* Some platforms have a broken implementation of
     * mbstowcs which does not count the characters that
     * would result from conversion.  Use an upper bound.
     */
    argsize = strlen(arg);
#else
    argsize = _Py_mbstowcs(NULL, arg, 0);
#endif
    if (argsize != DECODE_ERROR) {
        if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) {
            return -1;
        }
        res = (wchar_t *)PyMem_RawMalloc((argsize + 1) * sizeof(wchar_t));
        if (!res) {
            return -1;
        }

        count = _Py_mbstowcs(res, arg, argsize + 1);
        if (count != DECODE_ERROR) {
            *wstr = res;
            if (wlen != NULL) {
                *wlen = count;
            }
            return 0;
        }
        PyMem_RawFree(res);
    }

    /* Conversion failed. Fall back to escaping with surrogateescape. */
#ifdef HAVE_MBRTOWC
    /* Try conversion with mbrtwoc (C99), and escape non-decodable bytes. */

    /* Overallocate; as multi-byte characters are in the argument, the
       actual output could use less memory. */
    argsize = strlen(arg) + 1;
    if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t)) {
        return -1;
    }
    res = (wchar_t*)PyMem_RawMalloc(argsize * sizeof(wchar_t));
    if (!res) {
        return -1;
    }

    in = (unsigned char*)arg;
    out = res;
    memset(&mbs, 0, sizeof mbs);
    while (argsize) {
        size_t converted = _Py_mbrtowc(out, (char*)in, argsize, &mbs);
        if (converted == 0) {
            /* Reached end of string; null char stored. */
            break;
        }

        if (converted == INCOMPLETE_CHARACTER) {
            /* Incomplete character. This should never happen,
               since we provide everything that we have -
               unless there is a bug in the C library, or I
               misunderstood how mbrtowc works. */
            goto decode_error;
        }

        if (converted == DECODE_ERROR) {
            if (!surrogateescape) {
                goto decode_error;
            }

            /* Decoding error. Escape as UTF-8b, and start over in the initial
               shift state. */
            *out++ = 0xdc00 + *in++;
            argsize--;
            memset(&mbs, 0, sizeof mbs);
            continue;
        }

        // _Py_mbrtowc() reject lone surrogate characters
        assert(!Py_UNICODE_IS_SURROGATE(*out));

        /* successfully converted some bytes */
        in += converted;
        argsize -= converted;
        out++;
    }
    if (wlen != NULL) {
        *wlen = out - res;
    }
    *wstr = res;
    return 0;

decode_error:
    PyMem_RawFree(res);
    if (wlen) {
        *wlen = in - (unsigned char*)arg;
    }
    if (reason) {
        *reason = "decoding error";
    }
    return -2;
#else   /* HAVE_MBRTOWC */
    /* Cannot use C locale for escaping; manually escape as if charset
       is ASCII (i.e. escape all bytes > 128. This will still roundtrip
       correctly in the locale's charset, which must be an ASCII superset. */
    return decode_ascii(arg, wstr, wlen, reason, errors);
#endif   /* HAVE_MBRTOWC */
}


/* Decode a byte string from the locale encoding.

   Use the strict error handler if 'surrogateescape' is zero.  Use the
   surrogateescape error handler if 'surrogateescape' is non-zero: undecodable
   bytes are decoded as characters in range U+DC80..U+DCFF. If a byte sequence
   can be decoded as a surrogate character, escape the bytes using the
   surrogateescape error handler instead of decoding them.

   On success, return 0 and write the newly allocated wide character string into
   *wstr (use PyMem_RawFree() to free the memory). If wlen is not NULL, write
   the number of wide characters excluding the null character into *wlen.

   On memory allocation failure, return -1.

   On decoding error, return -2. If wlen is not NULL, write the start of
   invalid byte sequence in the input string into *wlen. If reason is not NULL,
   write the decoding error message into *reason.

   Return -3 if the error handler 'errors' is not supported.

   Use the Py_EncodeLocaleEx() function to encode the character string back to
   a byte string. */
int
_Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen,
                   const char **reason,
                   int current_locale, _Py_error_handler errors)
{
    if (current_locale) {
#ifdef _Py_FORCE_UTF8_LOCALE
        return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason,
                                errors);
#else
        return decode_current_locale(arg, wstr, wlen, reason, errors);
#endif
    }

#ifdef _Py_FORCE_UTF8_FS_ENCODING
    return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason,
                            errors);
#else
    int use_utf8 = (Py_UTF8Mode == 1);
#ifdef MS_WINDOWS
    use_utf8 |= !Py_LegacyWindowsFSEncodingFlag;
#endif
    if (use_utf8) {
        return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason,
                                errors);
    }

#ifdef USE_FORCE_ASCII
    if (force_ascii == -1) {
        force_ascii = check_force_ascii();
    }

    if (force_ascii) {
        /* force ASCII encoding to workaround mbstowcs() issue */
        return decode_ascii(arg, wstr, wlen, reason, errors);
    }
#endif

    return decode_current_locale(arg, wstr, wlen, reason, errors);
#endif   /* !_Py_FORCE_UTF8_FS_ENCODING */
}


/* Decode a byte string from the locale encoding with the
   surrogateescape error handler: undecodable bytes are decoded as characters
   in range U+DC80..U+DCFF. If a byte sequence can be decoded as a surrogate
   character, escape the bytes using the surrogateescape error handler instead
   of decoding them.

   Return a pointer to a newly allocated wide character string, use
   PyMem_RawFree() to free the memory. If size is not NULL, write the number of
   wide characters excluding the null character into *size

   Return NULL on decoding error or memory allocation error. If *size* is not
   NULL, *size is set to (size_t)-1 on memory error or set to (size_t)-2 on
   decoding error.

   Decoding errors should never happen, unless there is a bug in the C
   library.

   Use the Py_EncodeLocale() function to encode the character string back to a
   byte string. */
wchar_t*
Py_DecodeLocale(const char* arg, size_t *wlen)
{
    wchar_t *wstr;
    int res = _Py_DecodeLocaleEx(arg, &wstr, wlen,
                                 NULL, 0,
                                 _Py_ERROR_SURROGATEESCAPE);
    if (res != 0) {
        assert(res != -3);
        if (wlen != NULL) {
            *wlen = (size_t)res;
        }
        return NULL;
    }
    return wstr;
}


static int
encode_current_locale(const wchar_t *text, char **str,
                      size_t *error_pos, const char **reason,
                      int raw_malloc, _Py_error_handler errors)
{
    const size_t len = wcslen(text);
    char *result = NULL, *bytes = NULL;
    size_t i, size, converted;
    wchar_t c, buf[2];

    int surrogateescape;
    if (get_surrogateescape(errors, &surrogateescape) < 0) {
        return -3;
    }

    /* The function works in two steps:
       1. compute the length of the output buffer in bytes (size)
       2. outputs the bytes */
    size = 0;
    buf[1] = 0;
    while (1) {
        for (i=0; i < len; i++) {
            c = text[i];
            if (c >= 0xdc80 && c <= 0xdcff) {
                if (!surrogateescape) {
                    goto encode_error;
                }
                /* UTF-8b surrogate */
                if (bytes != NULL) {
                    *bytes++ = c - 0xdc00;
                    size--;
                }
                else {
                    size++;
                }
                continue;
            }
            else {
                buf[0] = c;
                if (bytes != NULL) {
                    converted = wcstombs(bytes, buf, size);
                }
                else {
                    converted = wcstombs(NULL, buf, 0);
                }
                if (converted == DECODE_ERROR) {
                    goto encode_error;
                }
                if (bytes != NULL) {
                    bytes += converted;
                    size -= converted;
                }
                else {
                    size += converted;
                }
            }
        }
        if (result != NULL) {
            *bytes = '\0';
            break;
        }

        size += 1; /* nul byte at the end */
        if (raw_malloc) {
            result = PyMem_RawMalloc(size);
        }
        else {
            result = PyMem_Malloc(size);
        }
        if (result == NULL) {
            return -1;
        }
        bytes = result;
    }
    *str = result;
    return 0;

encode_error:
    if (raw_malloc) {
        PyMem_RawFree(result);
    }
    else {
        PyMem_Free(result);
    }
    if (error_pos != NULL) {
        *error_pos = i;
    }
    if (reason) {
        *reason = "encoding error";
    }
    return -2;
}


/* Encode a string to the locale encoding.

   Parameters:

   * raw_malloc: if non-zero, allocate memory using PyMem_RawMalloc() instead
     of PyMem_Malloc().
   * current_locale: if non-zero, use the current LC_CTYPE, otherwise use
     Python filesystem encoding.
   * errors: error handler like "strict" or "surrogateescape".

   Return value:

    0: success, *str is set to a newly allocated decoded string.
   -1: memory allocation failure
   -2: encoding error, set *error_pos and *reason (if set).
   -3: the error handler 'errors' is not supported.
 */
static int
encode_locale_ex(const wchar_t *text, char **str, size_t *error_pos,
                 const char **reason,
                 int raw_malloc, int current_locale, _Py_error_handler errors)
{
    if (current_locale) {
#ifdef _Py_FORCE_UTF8_LOCALE
        return _Py_EncodeUTF8Ex(text, str, error_pos, reason,
                                raw_malloc, errors);
#else
        return encode_current_locale(text, str, error_pos, reason,
                                     raw_malloc, errors);
#endif
    }

#ifdef _Py_FORCE_UTF8_FS_ENCODING
    return _Py_EncodeUTF8Ex(text, str, error_pos, reason,
                            raw_malloc, errors);
#else
    int use_utf8 = (Py_UTF8Mode == 1);
#ifdef MS_WINDOWS
    use_utf8 |= !Py_LegacyWindowsFSEncodingFlag;
#endif
    if (use_utf8) {
        return _Py_EncodeUTF8Ex(text, str, error_pos, reason,
                                raw_malloc, errors);
    }

#ifdef USE_FORCE_ASCII
    if (force_ascii == -1) {
        force_ascii = check_force_ascii();
    }

    if (force_ascii) {
        return encode_ascii(text, str, error_pos, reason,
                            raw_malloc, errors);
    }
#endif

    return encode_current_locale(text, str, error_pos, reason,
                                 raw_malloc, errors);
#endif   /* _Py_FORCE_UTF8_FS_ENCODING */
}

static char*
encode_locale(const wchar_t *text, size_t *error_pos,
              int raw_malloc, int current_locale)
{
    char *str;
    int res = encode_locale_ex(text, &str, error_pos, NULL,
                               raw_malloc, current_locale,
                               _Py_ERROR_SURROGATEESCAPE);
    if (res != -2 && error_pos) {
        *error_pos = (size_t)-1;
    }
    if (res != 0) {
        return NULL;
    }
    return str;
}

/* Encode a wide character string to the locale encoding with the
   surrogateescape error handler: surrogate characters in the range
   U+DC80..U+DCFF are converted to bytes 0x80..0xFF.

   Return a pointer to a newly allocated byte string, use PyMem_Free() to free
   the memory. Return NULL on encoding or memory allocation error.

   If error_pos is not NULL, *error_pos is set to (size_t)-1 on success, or set
   to the index of the invalid character on encoding error.

   Use the Py_DecodeLocale() function to decode the bytes string back to a wide
   character string. */
char*
Py_EncodeLocale(const wchar_t *text, size_t *error_pos)
{
    return encode_locale(text, error_pos, 0, 0);
}


/* Similar to Py_EncodeLocale(), but result must be freed by PyMem_RawFree()
   instead of PyMem_Free(). */
char*
_Py_EncodeLocaleRaw(const wchar_t *text, size_t *error_pos)
{
    return encode_locale(text, error_pos, 1, 0);
}


int
_Py_EncodeLocaleEx(const wchar_t *text, char **str,
                   size_t *error_pos, const char **reason,
                   int current_locale, _Py_error_handler errors)
{
    return encode_locale_ex(text, str, error_pos, reason, 1,
                            current_locale, errors);
}


// Get the current locale encoding name:
//
// - Return "UTF-8" if _Py_FORCE_UTF8_LOCALE macro is defined (ex: on Android)
// - Return "UTF-8" if the UTF-8 Mode is enabled
// - On Windows, return the ANSI code page (ex: "cp1250")
// - Return "UTF-8" if nl_langinfo(CODESET) returns an empty string.
// - Otherwise, return nl_langinfo(CODESET).
//
// Return NULL on memory allocation failure.
//
// See also config_get_locale_encoding()
wchar_t*
_Py_GetLocaleEncoding(void)
{
#ifdef _Py_FORCE_UTF8_LOCALE
    // On Android langinfo.h and CODESET are missing,
    // and UTF-8 is always used in mbstowcs() and wcstombs().
    return _PyMem_RawWcsdup(L"UTF-8");
#else
    const PyPreConfig *preconfig = &_PyRuntime.preconfig;
    if (preconfig->utf8_mode) {
        return _PyMem_RawWcsdup(L"UTF-8");
    }

#ifdef MS_WINDOWS
    wchar_t encoding[23];
    unsigned int ansi_codepage = GetACP();
    swprintf(encoding, Py_ARRAY_LENGTH(encoding), L"cp%u", ansi_codepage);
    encoding[Py_ARRAY_LENGTH(encoding) - 1] = 0;
    return _PyMem_RawWcsdup(encoding);
#else
    const char *encoding = nl_langinfo(CODESET);
    if (!encoding || encoding[0] == '\0') {
        // Use UTF-8 if nl_langinfo() returns an empty string. It can happen on
        // macOS if the LC_CTYPE locale is not supported.
        return _PyMem_RawWcsdup(L"UTF-8");
    }

    wchar_t *wstr;
    int res = decode_current_locale(encoding, &wstr, NULL,
                                    NULL, _Py_ERROR_SURROGATEESCAPE);
    if (res < 0) {
        return NULL;
    }
    return wstr;
#endif  // !MS_WINDOWS

#endif  // !_Py_FORCE_UTF8_LOCALE
}


PyObject *
_Py_GetLocaleEncodingObject(void)
{
    wchar_t *encoding = _Py_GetLocaleEncoding();
    if (encoding == NULL) {
        PyErr_NoMemory();
        return NULL;
    }

    PyObject *str = PyUnicode_FromWideChar(encoding, -1);
    PyMem_RawFree(encoding);
    return str;
}

#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION

/* Check whether current locale uses Unicode as internal wchar_t form. */
int
_Py_LocaleUsesNonUnicodeWchar(void)
{
    /* Oracle Solaris uses non-Unicode internal wchar_t form for
       non-Unicode locales and hence needs conversion to UTF first. */
    char* codeset = nl_langinfo(CODESET);
    if (!codeset) {
        return 0;
    }
    /* 646 refers to ISO/IEC 646 standard that corresponds to ASCII encoding */
    return (strcmp(codeset, "UTF-8") != 0 && strcmp(codeset, "646") != 0);
}

static wchar_t *
_Py_ConvertWCharForm(const wchar_t *source, Py_ssize_t size,
                     const char *tocode, const char *fromcode)
{
    Py_BUILD_ASSERT(sizeof(wchar_t) == 4);

    /* Ensure we won't overflow the size. */
    if (size > (PY_SSIZE_T_MAX / (Py_ssize_t)sizeof(wchar_t))) {
        PyErr_NoMemory();
        return NULL;
    }

    /* the string doesn't have to be NULL terminated */
    wchar_t* target = PyMem_Malloc(size * sizeof(wchar_t));
    if (target == NULL) {
        PyErr_NoMemory();
        return NULL;
    }

    iconv_t cd = iconv_open(tocode, fromcode);
    if (cd == (iconv_t)-1) {
        PyErr_Format(PyExc_ValueError, "iconv_open() failed");
        PyMem_Free(target);
        return NULL;
    }

    char *inbuf = (char *) source;
    char *outbuf = (char *) target;
    size_t inbytesleft = sizeof(wchar_t) * size;
    size_t outbytesleft = inbytesleft;

    size_t ret = iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
    if (ret == DECODE_ERROR) {
        PyErr_Format(PyExc_ValueError, "iconv() failed");
        PyMem_Free(target);
        iconv_close(cd);
        return NULL;
    }

    iconv_close(cd);
    return target;
}

/* Convert a wide character string to the UCS-4 encoded string. This
   is necessary on systems where internal form of wchar_t are not Unicode
   code points (e.g. Oracle Solaris).

   Return a pointer to a newly allocated string, use PyMem_Free() to free