diff --git a/.azure-pipelines/windows-layout-steps.yml b/.azure-pipelines/windows-layout-steps.yml index afd897817904947..e15729fac3443db 100644 --- a/.azure-pipelines/windows-layout-steps.yml +++ b/.azure-pipelines/windows-layout-steps.yml @@ -12,7 +12,7 @@ steps: displayName: Show layout info (${{ parameters.kind }}) - ${{ if eq(parameters.fulltest, 'true') }}: - - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml" --tempdir "$(Build.BinariesDirectory)\tmp-${{ parameters.kind }}-$(arch)" -i test_launcher + - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml" --tempdir "$(Build.BinariesDirectory)\tmp-${{ parameters.kind }}-$(arch)" workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch) displayName: ${{ parameters.kind }} Tests env: diff --git a/.gitattributes b/.gitattributes index 4d52f900b93179e..f00d82e6978c40b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -19,7 +19,7 @@ *.zip binary # Specific binary files -PC/classicAppCompat.* binary +# -- None right now -- # Text files that should not be subject to eol conversion [attr]noeol -text diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0cdf17838b5798f..fd929b0c468699b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -188,12 +188,8 @@ Tools/wasm/wasi @brettcannon @emmatyping @savannahostrowski PC/ @python/windows-team PCbuild/ @python/windows-team -# Windows installer packages -Tools/msi/ @python/windows-team -Tools/nuget/ @python/windows-team - -# Windows Launcher -PC/launcher.c @python/windows-team @vsajip +# Windows Venv launcher/redirector +PC/venvlauncher.c @python/windows-team @vsajip # ---------------------------------------------------------------------------- diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d247f99aefe174c..8d4bd4a346f2e68 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -181,22 +181,6 @@ jobs: free-threading: ${{ matrix.free-threading }} interpreter: ${{ matrix.interpreter }} - build-windows-msi: - # ${{ '' } is a hack to nest jobs under the same sidebar category. - name: Windows MSI${{ '' }} # zizmor: ignore[obfuscation] - needs: build-context - if: fromJSON(needs.build-context.outputs.run-windows-msi) - strategy: - fail-fast: false - matrix: - arch: - - x86 - - x64 - - arm64 - uses: ./.github/workflows/reusable-windows-msi.yml - with: - arch: ${{ matrix.arch }} - build-macos: name: >- macOS @@ -650,7 +634,6 @@ jobs: - check-generated-files - check-c-api-docs - build-windows - - build-windows-msi - build-macos - build-ubuntu - build-ubuntu-ssltests @@ -671,7 +654,6 @@ jobs: allowed-failures: >- build-android, build-emscripten, - build-windows-msi, build-ubuntu-ssltests, test-hypothesis, cifuzz, diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index 6595aef0377e500..c998cbff181dd12 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -47,9 +47,6 @@ on: # yamllint disable-line rule:truthy run-wasi: description: Whether to run the WASI tests value: ${{ jobs.compute-changes.outputs.run-wasi }} # bool - run-windows-msi: - description: Whether to run the MSI installer smoke tests - value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool run-windows-tests: description: Whether to run the Windows tests value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool @@ -73,7 +70,6 @@ jobs: run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }} run-emscripten: ${{ steps.changes.outputs.run-emscripten }} run-wasi: ${{ steps.changes.outputs.run-wasi }} - run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} steps: - name: Set up Python diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml deleted file mode 100644 index d57046ae5ca8f00..000000000000000 --- a/.github/workflows/reusable-windows-msi.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Reusable Windows MSI - -on: - workflow_call: - inputs: - arch: - description: CPU architecture - required: true - type: string - -permissions: - contents: read - -env: - FORCE_COLOR: 1 - -jobs: - build: - name: installer for ${{ inputs.arch }} - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025' }} - timeout-minutes: 60 - env: - ARCH: ${{ inputs.arch }} - IncludeFreethreaded: true - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - - name: Build CPython installer - run: ./Tools/msi/build.bat --doc -"${ARCH}" - shell: bash diff --git a/.gitignore b/.gitignore index 118eb5ee76e8051..81b6bf967d30074 100644 --- a/.gitignore +++ b/.gitignore @@ -50,7 +50,7 @@ gmon.out *.exe -# Ignore core dumps... but not Tools/msi/core/ or the like. +# Ignore core dumps... but not .../core/ subdirectories core !core/ @@ -124,10 +124,14 @@ PCbuild/*-pgo PCbuild/*.VC.db PCbuild/*.VC.opendb PCbuild/amd64/ +PCbuild/amd64t/ PCbuild/arm32/ +PCbuild/arm32t/ PCbuild/arm64/ +PCbuild/arm64t/ PCbuild/obj/ PCbuild/win32/ +PCbuild/win32t/ Tools/unicode/data/ /autom4te.cache /build/ @@ -158,7 +162,6 @@ Tools/unicode/data/ /coverage/ /externals/ /htmlcov/ -Tools/msi/obj Tools/ssl/amd64 Tools/ssl/win32 Tools/freeze/test/outdir diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 78c95791ab31041..d04074f6e6a7993 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -499,12 +499,12 @@ Querying the error indicator .. c:function:: void PyErr_SetRaisedException(PyObject *exc) Set *exc* as the exception currently being raised, - clearing the existing exception if one is set. + clearing the existing exception if one is set. If *exc* is ``NULL``, + just clear the existing exception. - .. warning:: + *exc* must be a valid exception or ``NULL``. - This call ":term:`steals `" a reference to *exc*, - which must be a valid exception. + This call ":term:`steals `" a reference to *exc*. .. versionadded:: 3.12 diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 4159ff6e5965fbd..a04ef4e422949b9 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -243,3 +243,62 @@ Unless using :pep:`523`, you will not need this. Return the currently executing line number, or -1 if there is no line number. .. versionadded:: 3.12 + + +.. c:var:: const PyTypeObject *PyUnstable_ExecutableKinds + + An array of executable kinds (executor types) for frames, used for internal + debugging and tracing. + + Tools like debuggers and profilers can use this to identify the type of execution + context associated with a frame (such as to filter out internal frames). + The entries are indexed by the following constants: + + .. list-table:: + :header-rows: 1 + :widths: auto + + * - Constant + - Description + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_SKIP + - The frame is internal (For example: inlined) and should be skipped by tools. + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_PY_FUNCTION + - The frame corresponds to a standard Python function. + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION + - The frame corresponds to a function defined in native code. + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR + - The frame corresponds to a method on a class instance. + + However, Python's C API lacks a function to read the executable kind from + a frame. Instead, use this recipe: + + .. code-block:: c + + int + get_executable_kind(PyFrameObject *frame) + { + _PyInterpreterFrame *f = frame->f_frame; + PyObject *exec = PyStackRef_AsPyObjectBorrow(f->f_executable); + + if (PyCode_Check(exec)) { + return PyUnstable_EXECUTABLE_KIND_PY_FUNCTION; + } + if (PyMethod_Check(exec)) { + return PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION; + } + if (Py_IS_TYPE(exec, &PyMethodDescr_Type)) { + return PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR; + } + + return PyUnstable_EXECUTABLE_KIND_SKIP; + } + + .. versionadded:: 3.13 + + +.. c:macro:: PyUnstable_EXECUTABLE_KINDS + + The number of entries in :c:data:`PyUnstable_ExecutableKinds`. + + .. versionadded:: 3.13 + diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index ee73c1c8adaa7b3..0a446f86e22eaf6 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -433,7 +433,7 @@ accessible to C code. They all work with the current interpreter thread's This function is safe to call before :c:func:`Py_Initialize`. When called after runtime initialization, existing audit hooks are notified and may silently abort the operation by raising an error subclassed from - :class:`Exception` (other errors will not be silenced). + :class:`RuntimeError` (other errors will not be silenced). The hook function is always called with an :term:`attached thread state` by the Python interpreter that raised the event. @@ -447,7 +447,7 @@ accessible to C code. They all work with the current interpreter thread's If the interpreter is initialized, this function raises an auditing event ``sys.addaudithook`` with no arguments. If any existing hooks raise an - exception derived from :class:`Exception`, the new hook will not be + exception derived from :class:`RuntimeError`, the new hook will not be added and the exception is cleared. As a result, callers cannot assume that their hook has been added unless they control all existing hooks. @@ -462,6 +462,11 @@ accessible to C code. They all work with the current interpreter thread's .. versionadded:: 3.8 + .. versionchanged:: 3.8.1 + + Exceptions derived from :class:`Exception` but not :class:`RuntimeError` + are no longer suppressed. + .. _processcontrol: diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 6b68c0d085589a1..2e48c56bd2c5a5b 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -37,6 +37,10 @@ Type Objects Clear the internal lookup cache. Return the current version tag. + .. versionchanged:: 3.16 + This function is now a no-op as the type cache is now implemented + per-type. It still returns the current version tag. + .. c:function:: unsigned long PyType_GetFlags(PyTypeObject* type) Return the :c:member:`~PyTypeObject.tp_flags` member of *type*. This function is primarily @@ -699,7 +703,7 @@ but need extra remarks for use as slots: .. soft-deprecated:: 3.15 - When not targetting older Python versions, pefer :c:macro:`!Py_tp_bases`. + When not targeting older Python versions, prefer :c:macro:`!Py_tp_bases`. The following slots do not correspond to public fields in the underlying structures: diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 16dcb880712d244..4000f8aa1d4c0c7 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -3051,7 +3051,7 @@ Buffer Object Structures * Resource cleanup when the counter reaches zero must be done atomically, as the final release may race with concurrent releases from other - threads and dellocation must only happen once. + threads and deallocation must only happen once. The exporter MUST use the :c:member:`~Py_buffer.internal` field to keep track of buffer-specific resources. This field is guaranteed to remain diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 6256bf7a1454a9a..bba5c7f8ecf7751 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -343,11 +343,125 @@ the same library that the Python runtime is using. :py:mod:`!ast` Python module, which exports these constants under the same names. + .. rubric:: Low-level flags + + The following flags and masks serve narrow needs of the standard + library and interactive interpreters. Code outside the standard + library rarely has a reason to use them. They are considered + implementation details and may change at any time. + + .. c:macro:: PyCF_ALLOW_INCOMPLETE_INPUT + + This flag is a private interface between the compiler and the + :mod:`codeop` module. Do not use it; its behavior is unsupported + and may change without warning. + + With this flag set, when compilation fails because the source text + ends where more input is expected, for example in the middle of an + indented block or an unterminated string literal, the error raised + is the undocumented ``_IncompleteInputError``, a subclass of + :exc:`SyntaxError`. The :mod:`codeop` module sets this flag, + together with :c:macro:`PyCF_DONT_IMPLY_DEDENT`, to tell input + that is incomplete apart from input with a real syntax error, so + that interactive interpreters know when to prompt for another + line instead of reporting an error. + + .. versionadded:: 3.11 + + .. c:macro:: PyCF_DONT_IMPLY_DEDENT + + By default, when compiling with the :c:var:`Py_single_input` start + symbol, reaching the end of the source text implicitly closes any + open indented blocks. With this flag set, open blocks are only + closed if the last line of the source ends with a newline; otherwise, + compilation fails with a :exc:`SyntaxError`: + + .. code-block:: c + + PyCompilerFlags flags = { + .cf_flags = 0, + .cf_feature_version = PY_MINOR_VERSION, + }; + const char *source = "if a:\n pass"; + + /* The "if" block is closed implicitly; + this returns a code object: */ + Py_CompileStringFlags(source, "", Py_single_input, &flags); + + /* With the flag, this fails with a SyntaxError, + because the last line does not end with a newline: */ + flags.cf_flags = PyCF_DONT_IMPLY_DEDENT; + Py_CompileStringFlags(source, "", Py_single_input, &flags); + + The :mod:`codeop` module uses this flag to detect incomplete + interactive input. While the user is still typing inside an + indented block, the source does not yet end with a newline, so it + fails to compile and the user is prompted for another line. + + .. c:macro:: PyCF_IGNORE_COOKIE + + Read the source text as UTF-8, ignoring its :pep:`263` encoding + declaration ("coding cookie"), if any: + + .. code-block:: c + + PyCompilerFlags flags = { + .cf_flags = 0, + .cf_feature_version = PY_MINOR_VERSION, + }; + const char *source = "# coding: latin-1\ns = '\xe9'\n"; + + /* The coding cookie is honored: byte 0xE9 is decoded as + Latin-1, and this returns a code object that sets s to "é": */ + Py_CompileStringFlags(source, "", Py_file_input, &flags); + + /* With the flag, the cookie is ignored and compilation fails + with a SyntaxError, because 0xE9 is not valid UTF-8: */ + flags.cf_flags = PyCF_IGNORE_COOKIE; + Py_CompileStringFlags(source, "", Py_file_input, &flags); + + The :func:`compile`, :func:`eval` and :func:`exec` built-in functions + set this flag when the source is a :class:`str` object, because they + pass the text to the parser encoded as UTF-8. + + .. c:macro:: PyCF_SOURCE_IS_UTF8 + + Mark the source text as known to be UTF-8 encoded. + The :func:`compile`, :func:`eval` and :func:`exec` built-in functions + set this flag, but it currently has no effect. + The "``PyCF``" flags above can be combined with "``CO_FUTURE``" flags such as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally selectable using :ref:`future statements `. See :ref:`c_codeobject_flags` for a complete list. + The following masks combine several flags: + + .. c:macro:: PyCF_MASK + + Bitmask of all ``CO_FUTURE`` flags (see :ref:`c_codeobject_flags`), + which select features normally enabled by + :ref:`future statements `. + When code compiled with a ``PyCompilerFlags *flags`` argument + contains a ``from __future__ import`` statement, the flag for the + imported feature is added to *flags*, so that code executed later + in the same context inherits it. + + .. c:macro:: PyCF_MASK_OBSOLETE + + Do not use this mask in new code. It is kept only so that old + code passing its flags to :func:`compile` keeps working. + + Bitmask of flags for obsolete future features that no longer + have any effect. + + .. c:macro:: PyCF_COMPILE_MASK + + Bitmask of all ``PyCF`` flags that change how the source is + compiled, such as :c:macro:`PyCF_ONLY_AST`. + The :func:`compile` built-in function uses this mask to validate + its *flags* argument. + .. _start-symbols: diff --git a/Doc/deprecations/pending-removal-in-3.20.rst b/Doc/deprecations/pending-removal-in-3.20.rst index 011565dfbb090d4..d771764502c9103 100644 --- a/Doc/deprecations/pending-removal-in-3.20.rst +++ b/Doc/deprecations/pending-removal-in-3.20.rst @@ -53,3 +53,12 @@ Pending removal in Python 3.20 * Creating instances of abstract AST nodes (such as :class:`ast.AST` or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. + +* :mod:`typing`: + + * It is deprecated to call :func:`isinstance` and :func:`issubclass` checks on + protocol classes that were not explicitly decorated with :func:`!runtime_checkable` + but that inherit from a runtime-checkable protocol class. + This will raise a :exc:`TypeError` in Python 3.20. + + (Contributed by Bartosz Sławecki in :gh:`132604`.) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 698f2117ff5c649..ff38ba395fba22c 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -152,10 +152,11 @@ Alpha, beta and release candidate versions have an additional suffix: In other words, all versions labeled *2.0aN* precede the versions labeled *2.0bN*, which precede versions labeled *2.0rcN*, and *those* precede 2.0. -You may also find version numbers with a "+" suffix, e.g. "2.2+". These are -unreleased versions, built directly from the CPython development repository. In -practice, after a final minor release is made, the version is incremented to the -next minor version, which becomes the "a0" version, e.g. "2.4a0". +You may also find version numbers with a "+dev" suffix, such as "3.15.0b3+dev". +These are unreleased versions, built directly from the CPython development +repository. After the first beta release is made, the version is +incremented to the next feature version, which becomes the "a0" version, +such as "3.16.0a0". See the `Developer's Guide `__ diff --git a/Doc/howto/abi3t-migration.rst b/Doc/howto/abi3t-migration.rst index c542efbdea8decc..ec78a33ed25820e 100644 --- a/Doc/howto/abi3t-migration.rst +++ b/Doc/howto/abi3t-migration.rst @@ -8,14 +8,14 @@ Migrating to Stable ABI for free threading (``abi3t``) Starting with the 3.15 release, CPython supports a variant of the Stable ABI that supports :term:`free-threaded ` Python: -Stable ABI for Free-Threaded Builds, or ``abi3t`` for short. +the Stable ABI for Free-Threaded Builds, or ``abi3t`` for short. This document describes how to adapt C API extensions to support free threading. Why do this =========== -The typical reason to use Stable ABI is to reduce the number of artifacts that -you need to build and distribute for each version of your library. +The typical reason to use the Stable ABI is to reduce the number of artifacts +that you need to build and distribute for each version of your library. Without the Stable ABI, you must build a separate shared library, and typically a *wheel* distribution, for each feature version of CPython you wish @@ -87,16 +87,16 @@ builds; even the 3.15+ ones that this table "attributes" to ``abi3t``.) Why *not* do this ----------------- -There are two main downsides to Stable ABI. +There are two main downsides to the Stable ABI. -First, you extension may become slower, since Stable ABI prioritizes +First, your extension may become slower, since the Stable ABI prioritizes compatibility over performance. The difference is usually not noticeable, and often can be mitigated by using the same source to build both a Stable ABI build and a few version-specific ones for "tier 1" CPython versions. Second, not all of the C API is available. -Extensions need to be ported to build for Stable ABI, which may be difficult +Extensions need to be ported to build for the Stable ABI, which may be difficult or, in rare cases, impossible. Specifically, ``abi3t`` requires APIs added in CPython 3.15. @@ -127,7 +127,7 @@ Prerequisites This guide assumes that you have an extension written directly in C (or C++), which you want to port to ``abi3t``. -If your extenstion uses a code generator (like Cython) or language binding +If your extension uses a code generator (like Cython) or language binding (like PyO3), it's best to wait until that tool has support for ``abi3t``. If you maintain such a tool, you might be able to adapt the instructions here for your tool. @@ -135,7 +135,7 @@ here for your tool. Non-free-threaded Stable ABI ---------------------------- -Your extension should support the Stable ABI (``abi3t``). +Your extension should support the non-free-threaded Stable ABI (``abi3``). If not, either port it first, or follow this guide but be prepared to fix issues it does not mention. @@ -183,7 +183,7 @@ following just after ``#include ``:: #error "abi3t define is not set!" #endif -This should result in a different error than "``abt3t`` define is not set". +This should result in a different error than "``abi3t`` define is not set". .. note:: @@ -705,7 +705,7 @@ Testing Note that when you build an extension compatible with multiple versions of CPython, you should always *test* it with each version it supports (for example, 3.15, 3.16, and so on). -Stable ABI only guarantees *ABI* compatibility; there may also be behavior +The Stable ABI only guarantees *ABI* compatibility; there may also be behavior changes -- both intentional ones (covered by :pep:`387`) and bugs. Be sure to run tests on both free-threaded and non-free-threaded builds diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 87025814aafb9ab..2ff221cf258eda5 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1955,7 +1955,7 @@ Subclass ``QueueListener`` class NNGSocketListener(logging.handlers.QueueListener): def __init__(self, uri, /, *handlers, **kwargs): - # Have a timeout for interruptability, and open a + # Have a timeout for interruptibility, and open a # subscriber socket socket = pynng.Sub0(listen=uri, recv_timeout=500) # The b'' subscription matches all topics diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index af28fe0e2fde2f8..a90754620b59fdf 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -535,9 +535,9 @@ following attributes: :attr:`~Format.VALUE_WITH_FAKE_GLOBALS`. * A :ref:`code object ` ``__code__`` containing the compiled code for the annotate function. -* Optional: A tuple of the function's positional defaults ``__kwdefaults__``, if the +* Optional: A tuple of the function's positional defaults ``__defaults__``, if the function represented by ``__code__`` uses any positional defaults. -* Optional: A dict of the function's keyword defaults ``__defaults__``, if the function +* Optional: A dict of the function's keyword defaults ``__kwdefaults__``, if the function represented by ``__code__`` uses any keyword defaults. * Optional: All other :ref:`function attributes `. diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 4809fdb42bf3d77..5bdfe7c182de3ac 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1366,7 +1366,7 @@ Control flow ``try`` blocks which are followed by ``except*`` clauses. The attributes are the same as for :class:`Try` but the :class:`ExceptHandler` nodes in ``handlers`` - are interpreted as ``except*`` blocks rather then ``except``. + are interpreted as ``except*`` blocks rather than ``except``. .. doctest:: diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 6b5244abe427cd3..d24c8420ef8920c 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -1873,6 +1873,12 @@ Do not instantiate the :class:`Server` class directly. Wait until the :meth:`close` method completes and all active connections have finished. + .. versionchanged:: 3.12 + ``wait_closed()`` now waits until the server is closed and + all active connections have finished. Previously, it returned + immediately if the server was already closed, even if + connections were still active. + .. attribute:: sockets List of socket-like objects, ``asyncio.trsock.TransportSocket``, which diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index a6514649bf9a0a8..70f711b779edf85 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -240,10 +240,34 @@ their completion. Note, that the data read is buffered in memory, so do not use this method if the data size is large or unlimited. + If this coroutine is cancelled (for example, when a timeout is + set with :func:`~asyncio.wait_for`), the output that was already + read is not lost: call :meth:`!communicate` again to read the + remaining output and get the complete data:: + + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(), timeout=5.0) + except TimeoutError: + proc.kill() + stdout, stderr = await proc.communicate() + + Passing *input* after a previous :meth:`!communicate` call was + cancelled raises :exc:`ValueError`; pass ``input=None`` to + continue the communication, the original *input* is used. + .. versionchanged:: 3.12 *stdin* gets closed when ``input=None`` too. + .. versionchanged:: next + + If :meth:`!communicate` is cancelled, the output that was + already read is now preserved and returned by a subsequent + :meth:`!communicate` call. Passing *input* to a + :meth:`!communicate` call following a cancelled one now raises + :exc:`ValueError`. + .. method:: send_signal(signal) Sends the signal *signal* to the child process. diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 60cd214b4501010..fb0ffc73b48c40c 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -400,9 +400,9 @@ For simple text calendars this module provides the following functions. *month* (``1``--``12``), *day* (``1``--``31``). -.. function:: weekheader(n) +.. function:: weekheader(width) - Return a header containing abbreviated weekday names. *n* specifies the width in + Return a header containing abbreviated weekday names. *width* specifies the width in characters for one weekday. @@ -430,12 +430,12 @@ For simple text calendars this module provides the following functions. of the :class:`TextCalendar` class. -.. function:: prcal(year, w=0, l=0, c=6, m=3) +.. function:: prcal(theyear, w=0, l=0, c=6, m=3) Prints the calendar for an entire year as returned by :func:`calendar`. -.. function:: calendar(year, w=2, l=1, c=6, m=3) +.. function:: calendar(theyear, w=2, l=1, c=6, m=3) Returns a 3-column calendar for an entire year as a multi-line string using the :meth:`~TextCalendar.formatyear` of the :class:`TextCalendar` class. diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst index 6d99e36e1e5bb65..6618ccf7d559e78 100644 --- a/Doc/library/compression.zstd.rst +++ b/Doc/library/compression.zstd.rst @@ -503,7 +503,7 @@ Advanced parameter control The :meth:`~.bounds` method can be used on any attribute to get the valid values for that parameter. - Parameters are optional; any omitted parameter will have it's value selected + Parameters are optional; any omitted parameter will have its value selected automatically. Example getting the lower and upper bound of :attr:`~.compression_level`:: @@ -732,7 +732,7 @@ Advanced parameter control An :class:`~enum.IntEnum` containing the advanced decompression parameter keys that can be used when decompressing data. Parameters are optional; any - omitted parameter will have it's value selected automatically. + omitted parameter will have its value selected automatically. The :meth:`~.bounds` method can be used on any attribute to get the valid values for that parameter. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 87c2d4702e74086..81949261c563d04 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -312,6 +312,28 @@ The :mod:`!csv` module defines the following classes: is given, it is interpreted as a string containing possible valid delimiter characters. + The dialect is deduced by parsing the sample with every plausible + combination of parameters + and choosing the combination which splits the sample into rows + with the most consistent number of fields, + so the returned dialect is consistent with how :func:`reader` + will parse the sample. + Raise :exc:`Error` if no combination fits the sample, + in particular if it is a single column, + so there is no delimiter to find. + + If several combinations fit the sample equally well --- + for example if both ``','`` and ``';'`` split every row consistently --- + the delimiters ``','``, ``'\t'``, ``';'``, ``' '`` and ``':'`` + are preferred, in this order, + no matter how many times each of them occurs. + + .. versionchanged:: next + The dialect is now deduced by trial parsing + and the results may differ from those of earlier Python versions. + The *escapechar* parameter can now be detected, + and the requested *delimiters* are not restricted to ASCII. + .. method:: has_header(sample) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 80868a8b2418f35..acd6c2f97348e19 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -694,6 +694,45 @@ through the :attr:`~_CFuncPtr.errcheck` attribute; see the reference manual for details. +Specifying function pointers using type annotations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. decorator:: wrap_dll_function(dll) + :module: ctypes.util + + A :term:`decorator` that generates :attr:`~ctypes._CFuncPtr.argtypes` and + :attr:`~ctypes._CFuncPtr.restype` from a function signature, using the + :attr:`~function.__name__` of the function and its :term:`type annotations `. + + The decorated function should look like this:: + + @wrap_dll_function(dll_to_wrap) + def function_ptr_name(arg_name: ctypes_type, ...) -> ctypes_type: + """Optional docstring. There should be no function body.""" + + The body of the decorated function is ignored, and any parameters that are + missing type annotations are skipped. The names of the parameters are ignored + and do not have to match the underlying C implementation. + + If the decorated function does not have a return type annotation, a + :exc:`ValueError` is raised. If the name of the function does not exist + in *dll*, an :exc:`AttributeError` is raised. + + For example:: + + import ctypes + from ctypes.util import wrap_dll_function + + @wrap_dll_function(ctypes.pythonapi) + def PyObject_GetAttrString(op: ctypes.py_object, attr: ctypes.c_char_p) -> ctypes.py_object: + pass + + PyObject_GetAttrString(42, b"real") + + + .. versionadded:: next + + .. _ctypes-passing-pointers: Passing pointers (or: passing parameters by reference) @@ -1827,6 +1866,10 @@ like ``find_library("c")`` will fail and return ``None``. .. availability:: Windows + .. soft-deprecated:: 3.16 + This function now always returns ``None``, as there are no more + VC runtime DLLs that are a single file and supported by Microsoft. + .. _ctypes-listing-loaded-shared-libraries: diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index e413cd9d2bef2d9..a833914a5d56d44 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -129,6 +129,8 @@ Initialization and termination and return the previously current screen. Returns ``None`` if the previous screen was the one created by :func:`initscr`. + Raises :exc:`error` if *screen* has no terminal, + as is the case for a screen returned by :func:`new_prescr`. .. versionadded:: next @@ -1321,6 +1323,8 @@ Reading window contents The bottom 8 bits are the character proper and the upper bits are the attributes; extract them with the :data:`A_CHARTEXT` and :data:`A_ATTRIBUTES` bit-masks, and the color pair with :func:`pair_number`. + The character byte is the locale-encoded byte of the cell's character, + consistent with :meth:`instr`. It cannot represent a cell holding combining characters, a character that does not fit in a single byte, or a color pair outside the :func:`color_pair` range; use :meth:`in_wch` for those, which returns it as a :class:`complexchar`. diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst index ff51b2779e5fa33..4b4f3b1720accc4 100644 --- a/Doc/library/datatypes.rst +++ b/Doc/library/datatypes.rst @@ -8,9 +8,10 @@ The modules described in this chapter provide a variety of specialized data types such as dates and times, fixed-type arrays, heap queues, double-ended queues, and enumerations. -Python also provides some built-in data types, in particular, -:class:`dict`, :class:`list`, :class:`set` and :class:`frozenset`, and -:class:`tuple`. The :class:`str` class is used to hold +Python also provides :ref:`some built-in data types `, in particular, +:class:`list`, :class:`tuple`, :class:`dict`, :class:`frozendict`, +:class:`set`, and :class:`frozenset`. +The :class:`str` class is used to hold Unicode strings, and the :class:`bytes` and :class:`bytearray` classes are used to hold binary data. diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 25edb40e35a630a..374af16b2b6f98b 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -13,45 +13,87 @@ -------------- -This module provides classes and functions for comparing sequences. It -can be used for example, for comparing files, and can produce information -about file differences in various formats, including HTML and context and unified -diffs. For comparing directories and files, see also, the :mod:`filecmp` module. - - -.. class:: SequenceMatcher - :noindex: - - This is a flexible class for comparing pairs of sequences of any type, so long - as the sequence elements are :term:`hashable`. The basic algorithm predates, and is a - little fancier than, an algorithm published in the late 1980's by Ratcliff and - Obershelp under the hyperbolic name "gestalt pattern matching." The idea is to - find the longest contiguous matching subsequence that contains no "junk" - elements; these "junk" elements are ones that are uninteresting in some - sense, such as blank lines or whitespace. (Handling junk is an - extension to the Ratcliff and Obershelp algorithm.) The same - idea is then applied recursively to the pieces of the sequences to the left and - to the right of the matching subsequence. This does not yield minimal edit - sequences, but does tend to yield matches that "look right" to people. - - **Timing:** The basic Ratcliff-Obershelp algorithm is cubic time in the worst - case and quadratic time in the expected case. :class:`SequenceMatcher` is - quadratic time for the worst case and has expected-case behavior dependent in a - complicated way on how many elements the sequences have in common; best case - time is linear. - - **Automatic junk heuristic:** :class:`SequenceMatcher` supports a heuristic that - automatically treats certain sequence items as junk. The heuristic counts how many - times each individual item appears in the sequence. If an item's duplicates (after - the first one) account for more than 1% of the sequence and the sequence is at least - 200 items long, this item is marked as "popular" and is treated as junk for - the purpose of sequence matching. This heuristic can be turned off by setting - the ``autojunk`` argument to ``False`` when creating the :class:`SequenceMatcher`. +This module provides classes and functions for comparing sequences. +Most of them compare sequences of text lines (for example lists of strings, +or :term:`file objects `) and +produce :dfn:`diffs` -- reports on the differences. +Diffs can be produced in in various formats, including HTML and context +and unified diffs -- formats produced by tools like +:manpage:`diff ` and :manpage:`git diff `. - .. versionchanged:: 3.2 - Added the *autojunk* parameter. +Comparisons are done using a matching algorithm implemented in +:class:`SequenceMatcher` -- a flexible class for comparing pairs of sequences +of any type, not just text, so long as the sequence elements are +:term:`hashable`. + + +.. _difflib-junk: + +Junk heuristic +-------------- + +:mod:`!difflib` uses a :dfn:`junk` heuristic: some items are deemed to be +:dfn:`junk`, and ignored when searching for similarities. +Ideally, these are uninteresting or common items, such as blank lines +or whitespace. + +This heuristic can speed the algorithm up (because it reduces the number of +possible combinations) and it can produce results that are more understandable +for humans (typically breaking on whitespace). +But it can also cause pathological cases: + +- Inappropriately chosen junk items can cause an unexpectedly **large** (but + still correct) result. +- The default heuristic is **asymmetric**: only the second sequence is + inspected when determining what is considered junk, so comparing A to B can + give different results than comparing B to A and reversing the result. + +By default, if the second input sequence is at least 200 items long, items +that account for more than 1% it are considered *junk*. + +Depending on your data, you should consider turning this heuristic off +(setting :class:`~difflib.SequenceMatcher`'s *autojunk* argument to to ``False``) +or tuning it (using the *isjunk* argument, perhaps to one of the +:ref:`predefined functions `). + + +The :mod:`!difflib` algorithm +----------------------------- + +The algorithm used in :class:`SequenceMatcher` predates, and is a little +fancier than, an algorithm published in the late 1980s by Ratcliff and +Obershelp under the hyperbolic name "gestalt pattern matching." +The idea is to find the longest contiguous subsequence common to both inputs, +then recursively handle the pieces of the sequences to the left and to the +right of the matching subsequence. + +.. seealso:: + + `Pattern Matching: The Gestalt Approach `_ + Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This + was published in Dr. Dobb's Journal in July, 1988. + +As an extension to the Ratcliff and Obershelp algorithm, :mod:`!difflib` +searches for the longest *junk-free* contiguous subsequence. +See the :ref:`difflib-junk` section for details. + +.. impl-detail:: Timing + + The basic Ratcliff-Obershelp algorithm is cubic time in the worst + case and quadratic time in the expected case. + :mod:`difflib`'s algorithm is quadratic time for the worst case and has + expected-case behavior dependent in a complicated way on how many elements + the sequences have in common; + best case time is linear. +.. _difflib-diff-generation: + +Diff generation +--------------- + +.. _differ-objects: + .. class:: Differ This is a class for comparing sequences of lines of text, and producing @@ -77,6 +119,47 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. and were not present in either input sequence. These lines can be confusing if the sequences contain whitespace characters, such as spaces, tabs or line breaks. + Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** + diffs. To the contrary, minimal diffs are often counter-intuitive for humans, + because they synch up anywhere possible, sometimes at accidental matches + 100 pages apart. + Restricting synch points to contiguous matches preserves some notion of + locality, at the occasional cost of producing a longer diff. + + The :class:`Differ` class has this constructor: + + .. method:: __init__(linejunk=None, charjunk=None) + + Optional keyword parameters *linejunk* and *charjunk* are for filter functions + (or ``None``): + + *linejunk*: A function that accepts a single string argument, and returns true + if the string is junk. The default is ``None``, meaning that no line is + considered junk. + + *charjunk*: A function that accepts a single character argument (a string of + length 1), and returns true if the character is junk. The default is ``None``, + meaning that no character is considered junk. + + These junk-filtering functions speed up matching to find + differences and do not cause any differing lines or characters to + be ignored. Read the description of the + :meth:`~SequenceMatcher.find_longest_match` method's *isjunk* + parameter for an explanation. + + :class:`Differ` objects are used (deltas generated) via a single method: + + + .. method:: Differ.compare(a, b) + + Compare two sequences of lines, and generate the delta (a sequence of lines). + + Each sequence must contain individual single-line strings ending with + newlines. Such sequences can be obtained from the + :meth:`~io.IOBase.readlines` method of file-like objects. The generated + delta also consists of newline-terminated strings, ready to be + printed as-is via the :meth:`~io.IOBase.writelines` method of a + file-like object. .. class:: HtmlDiff @@ -85,6 +168,11 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. with inter-line and intra-line change highlights. The table can be generated in either full or contextual difference mode. + .. warning:: + + The trailing newlines get stripped before the diff, so the result can be + incomplete. See :gh:`71896` for details. + The constructor for this class is: @@ -339,6 +427,12 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. versionadded:: 3.5 + +.. _difflib-isjunk-functions: + +Junk definition functions +------------------------- + .. function:: IS_LINE_JUNK(line) Return ``True`` for ignorable lines. The line *line* is ignorable if *line* is @@ -353,21 +447,11 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. parameter *charjunk* in :func:`ndiff`. -.. seealso:: - - `Pattern Matching: The Gestalt Approach `_ - Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This - was published in Dr. Dobb's Journal in July, 1988. - - .. _sequence-matcher: SequenceMatcher objects ----------------------- -The :class:`SequenceMatcher` class has this constructor: - - .. class:: SequenceMatcher(isjunk=None, a='', b='', autojunk=True) Optional argument *isjunk* must be ``None`` (the default) or a one-argument @@ -553,16 +637,6 @@ The :class:`SequenceMatcher` class has this constructor: to try :meth:`quick_ratio` or :meth:`real_quick_ratio` first to get an upper bound. - .. note:: - - Caution: The result of a :meth:`ratio` call may depend on the order of - the arguments. For instance:: - - >>> SequenceMatcher(None, 'tide', 'diet').ratio() - 0.25 - >>> SequenceMatcher(None, 'diet', 'tide').ratio() - 0.5 - .. method:: quick_ratio() @@ -588,10 +662,13 @@ are always at least as large as :meth:`~SequenceMatcher.ratio`: 1.0 +Examples +-------- + .. _sequencematcher-examples: SequenceMatcher examples ------------------------- +........................ This example compares two strings, considering blanks to be "junk": @@ -639,59 +716,10 @@ If you want to know how to change the first sequence into the second, use built with :class:`SequenceMatcher`. -.. _differ-objects: - -Differ objects --------------- - -Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** -diffs. To the contrary, minimal diffs are often counter-intuitive, because they -synch up anywhere possible, sometimes accidental matches 100 pages apart. -Restricting synch points to contiguous matches preserves some notion of -locality, at the occasional cost of producing a longer diff. - -The :class:`Differ` class has this constructor: - - -.. class:: Differ(linejunk=None, charjunk=None) - :noindex: - - Optional keyword parameters *linejunk* and *charjunk* are for filter functions - (or ``None``): - - *linejunk*: A function that accepts a single string argument, and returns true - if the string is junk. The default is ``None``, meaning that no line is - considered junk. - - *charjunk*: A function that accepts a single character argument (a string of - length 1), and returns true if the character is junk. The default is ``None``, - meaning that no character is considered junk. - - These junk-filtering functions speed up matching to find - differences and do not cause any differing lines or characters to - be ignored. Read the description of the - :meth:`~SequenceMatcher.find_longest_match` method's *isjunk* - parameter for an explanation. - - :class:`Differ` objects are used (deltas generated) via a single method: - - - .. method:: Differ.compare(a, b) - - Compare two sequences of lines, and generate the delta (a sequence of lines). - - Each sequence must contain individual single-line strings ending with - newlines. Such sequences can be obtained from the - :meth:`~io.IOBase.readlines` method of file-like objects. The delta - generated also consists of newline-terminated strings, ready to be - printed as-is via the :meth:`~io.IOBase.writelines` method of a - file-like object. - - .. _differ-examples: Differ example --------------- +.............. This example compares two texts. First we set up the texts, sequences of individual single-line strings ending with newlines (such sequences can also be @@ -758,14 +786,14 @@ As a single multi-line string it looks like this:: .. _difflib-interface: A command-line interface to difflib ------------------------------------ +................................... This example shows how to use difflib to create a ``diff``-like utility. .. literalinclude:: ../includes/diff.py ndiff example -------------- +............. This example shows how to use :func:`difflib.ndiff`. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 3775d5ac81a2736..ecf62fb6391b1b1 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -215,6 +215,8 @@ The following exceptions are the exceptions that are usually raised. The object that was accessed for the named attribute. + When possible, :attr:`name` and :attr:`obj` are set automatically. + .. versionchanged:: 3.10 Added the :attr:`name` and :attr:`obj` attributes. diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index a4d37beb436899a..f45ab397e936938 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -611,9 +611,10 @@ are always available. They are listed here in alphabetical order. untrusted user-supplied input will lead to security vulnerabilities. The *source* argument is parsed and evaluated as a Python expression - (technically speaking, a condition list) using the *globals* and *locals* - mappings as global and local namespace. If the *globals* dictionary is - present and does not contain a value for the key ``__builtins__``, a + (technically speaking, an :ref:`expression list `) + using the *globals* and *locals* mappings as global and local namespace. + If the *globals* dictionary is present and does not contain a value for the + key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is inserted under that key before *source* is parsed. Overriding ``__builtins__`` can be used to restrict or change the available @@ -633,6 +634,9 @@ are always available. They are listed here in alphabetical order. >>> eval('x+1') 2 + >>> eval("1, 2") + (1, 2) + This function can also be used to execute arbitrary code objects (such as those created by :func:`compile`). In this case, pass a code object instead of a string. If the code object has been compiled with ``'exec'`` as the @@ -2175,6 +2179,11 @@ are always available. They are listed here in alphabetical order. in the same way that keywords in a class definition (besides *metaclass*) would. + Unlike a :keyword:`class` statement, the three argument form does not + call the metaclass ``__prepare__`` method (see :ref:`prepare`). Use + :func:`types.new_class` to dynamically create a class using the + appropriate metaclass. + See also :ref:`class-customization`. .. versionchanged:: 3.6 diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst index 5c63009e22d58c4..9995f114d063e82 100644 --- a/Doc/library/getopt.rst +++ b/Doc/library/getopt.rst @@ -62,6 +62,8 @@ exception: option ``--fo`` will match as ``--foo``, but ``--f`` will not match uniquely, so :exc:`GetoptError` will be raised. + If *longopts* is a string it gets treated as a list of a single element. + The return value consists of two elements: the first is a list of ``(option, value)`` pairs; the second is the list of program arguments left after the option list was stripped (this is a trailing slice of *args*). Each diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index a0f7379b12a8a62..eddc98824d48106 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -635,8 +635,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: ismethoddescriptor(object) Return ``True`` if the object is a method descriptor, but not if - :func:`ismethod`, :func:`isclass`, :func:`isfunction` or :func:`isbuiltin` - are true. + :func:`isclass`, :func:`ismethod` or :func:`isfunction` is true. This, for example, is true of ``int.__add__``. An object passing this test has a :meth:`~object.__get__` method, but not a :meth:`~object.__set__` @@ -644,10 +643,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): attributes varies. A :attr:`~definition.__name__` attribute is usually sensible, and :attr:`~definition.__doc__` often is. - Methods implemented via descriptors that also pass one of the other tests - return ``False`` from the :func:`ismethoddescriptor` test, simply because the - other tests promise more -- you can, e.g., count on having the - :attr:`~method.__func__` attribute (etc) when an object passes + Method descriptors that also pass any of the other tests (:func:`!isclass`, + :func:`!ismethod` or :func:`!isfunction`) make this function return ``False``, + simply because those other tests promise more -- you can, for example, count + on having the :attr:`~method.__func__` attribute when an object passes :func:`ismethod`. .. versionchanged:: 3.13 @@ -658,16 +657,28 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: isdatadescriptor(object) - Return ``True`` if the object is a data descriptor. + Return ``True`` if the object is a data descriptor, but not if + :func:`isclass`, :func:`ismethod` or :func:`isfunction` is true. - Data descriptors have a :attr:`~object.__set__` or a :attr:`~object.__delete__` method. - Examples are properties (defined in Python), getsets, and members. The - latter two are defined in C and there are more specific tests available for - those types, which is robust across Python implementations. Typically, data - descriptors will also have :attr:`~definition.__name__` and :attr:`!__doc__` attributes - (properties, getsets, and members have both of these attributes), but this is - not guaranteed. + Data descriptors always have a :meth:`~object.__set__` method and/or + a :meth:`~object.__delete__` method. Optionally, they may also have a + :meth:`~object.__get__` method. + Examples of data descriptors are :func:`properties `, getsets and + member descriptors. Note that for the latter two (defined only in C extension + modules), more specific tests are available: :func:`isgetsetdescriptor` and + :func:`ismemberdescriptor`, respectively. + + While data descriptors may also have :attr:`~definition.__name__` and + :attr:`!__doc__` attributes (as properties, getsets and member descriptors + do), this is not necessarily the case in general. + + .. versionchanged:: 3.8 + This function now reports objects with only a :meth:`~object.__set__` method + as being data descriptors (the presence of :meth:`~object.__get__` is no + longer required for that). Moreover, objects with :meth:`~object.__delete__`, + but not :meth:`~object.__set__`, are now properly recognized as data + descriptors as well, which was not the case previously. .. function:: isgetsetdescriptor(object) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 5152c7561fa1f26..7230224b4fc5323 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -631,7 +631,8 @@ supports sending logging messages to a remote or local Unix syslog. the form of a ``(host, port)`` tuple. If *address* is not specified, ``('localhost', 514)`` is used. The address is used to open a socket. An alternative to providing a ``(host, port)`` tuple is providing an address as a - string, for example '/dev/log'. In this case, a Unix domain socket is used to + string or a :class:`bytes` object, for example '/dev/log'. + In this case, a Unix domain socket is used to send the message to the syslog. If *facility* is not specified, :const:`LOG_USER` is used. The type of socket opened depends on the *socktype* argument, which defaults to :const:`socket.SOCK_DGRAM` and thus @@ -664,6 +665,9 @@ supports sending logging messages to a remote or local Unix syslog. .. versionchanged:: 3.14 *timeout* was added. + .. versionchanged:: next + *address* can now be a :class:`bytes` object. + .. method:: close() Closes the socket to the remote host. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index cd72174d54f6e62..580829cfbb46e07 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -152,35 +152,14 @@ Compressing and decompressing data in memory :func:`compress`. The *format* argument specifies what container format should be used. - Possible values are: - - * :const:`FORMAT_XZ`: The ``.xz`` container format. - This is the default format. - - * :const:`FORMAT_ALONE`: The legacy ``.lzma`` container format. - This format is more limited than ``.xz`` -- it does not support integrity - checks or multiple filters. - - * :const:`FORMAT_RAW`: A raw data stream, not using any container format. - This format specifier does not support integrity checks, and requires that - you always specify a custom filter chain (for both compression and - decompression). Additionally, data compressed in this manner cannot be - decompressed using :const:`FORMAT_AUTO` (see :class:`LZMADecompressor`). + Possible values are :const:`FORMAT_XZ` (the default), + :const:`FORMAT_ALONE` and :const:`FORMAT_RAW`. The *check* argument specifies the type of integrity check to include in the compressed data. This check is used when decompressing, to ensure that the - data has not been corrupted. Possible values are: - - * :const:`CHECK_NONE`: No integrity check. - This is the default (and the only acceptable value) for - :const:`FORMAT_ALONE` and :const:`FORMAT_RAW`. - - * :const:`CHECK_CRC32`: 32-bit Cyclic Redundancy Check. - - * :const:`CHECK_CRC64`: 64-bit Cyclic Redundancy Check. - This is the default for :const:`FORMAT_XZ`. - - * :const:`CHECK_SHA256`: 256-bit Secure Hash Algorithm. + data has not been corrupted. Possible values are :const:`CHECK_NONE`, + :const:`CHECK_CRC32`, :const:`CHECK_CRC64` (the default for + :const:`FORMAT_XZ`) and :const:`CHECK_SHA256`. If the specified check is not supported, an :class:`LZMAError` is raised. @@ -412,6 +391,106 @@ These filters support one option, ``start_offset``. This specifies the address that should be mapped to the beginning of the input data. The default is 0. +Constants +--------- + +The following module-level constants are provided for use as the *format*, +*check*, *preset* and *filters* arguments of the classes and functions above. + +Container formats: + +.. data:: FORMAT_XZ + + The ``.xz`` container format. + +.. data:: FORMAT_ALONE + + The legacy ``.lzma`` container format. This format is more limited than + ``.xz`` -- it does not support integrity checks or multiple filters. + +.. data:: FORMAT_RAW + + A raw data stream, not using any container format. This format specifier + does not support integrity checks, and requires that you always specify a + custom filter chain (for both compression and decompression). Additionally, + data compressed in this manner cannot be decompressed using + :const:`FORMAT_AUTO`. + +.. data:: FORMAT_AUTO + + Used for decompression only. The container format is detected + automatically, so that both ``.xz`` and ``.lzma`` files can be decompressed. + +Integrity checks: + +.. data:: CHECK_NONE + + No integrity check. This is the default (and the only acceptable value) for + :const:`FORMAT_ALONE` and :const:`FORMAT_RAW`. + +.. data:: CHECK_CRC32 + + A 32-bit Cyclic Redundancy Check. + +.. data:: CHECK_CRC64 + + A 64-bit Cyclic Redundancy Check. This is the default for + :const:`FORMAT_XZ`. + +.. data:: CHECK_SHA256 + + A 256-bit Secure Hash Algorithm. + +.. data:: CHECK_UNKNOWN + + The integrity check used by a stream could not yet be determined. This may + be the value of the :attr:`LZMADecompressor.check` attribute until enough of + the input has been decoded. + +.. data:: CHECK_ID_MAX + + The largest supported integrity-check ID. + +Compression presets: + +.. data:: PRESET_DEFAULT + + The default compression preset, equivalent to preset level ``6``. + +.. data:: PRESET_EXTREME + + A flag that may be bitwise OR-ed with a preset level (``0`` to ``9``) to + select a slower but more thorough variant of that preset. + +Filter IDs and options: + +.. data:: FILTER_LZMA1 + FILTER_LZMA2 + + The LZMA1 and LZMA2 compression filters. :const:`FILTER_LZMA1` is for use + with :const:`FORMAT_ALONE`, while :const:`FILTER_LZMA2` is for use with + :const:`FORMAT_XZ` and :const:`FORMAT_RAW`. + +.. data:: FILTER_DELTA + + The delta filter. + +.. data:: MODE_FAST + MODE_NORMAL + + Compression modes that may be used as the ``mode`` option of a filter + specifier (see :ref:`filter-chain-specs`). + +.. data:: MF_HC3 + MF_HC4 + MF_BT2 + MF_BT3 + MF_BT4 + + Match finders that may be used as the ``mf`` option of a filter specifier + (see :ref:`filter-chain-specs`). + + Examples -------- diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 5c29fff146eef00..f6cfbb5126158e1 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -116,12 +116,13 @@ behavior of the module. Previously, Windows registry settings were ignored. -.. function:: read_mime_types(filename) +.. function:: read_mime_types(file) - Load the type map given in the file *filename*, if it exists. The type map is - returned as a dictionary mapping filename extensions, including the leading dot - (``'.'``), to strings of the form ``'type/subtype'``. If the file *filename* - does not exist or cannot be read, ``None`` is returned. + Load the type map given in the file named by *file*, if it exists. *file* + must be a string specifying the name of the file to read. The type map is + returned as a dictionary mapping file extensions, including the leading dot + (``'.'``), to strings of the form ``'type/subtype'``. If the file does not + exist or cannot be read, ``None`` is returned. .. function:: add_type(type, ext, strict=True) diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 6b49c1a9ccd6e11..f4a1125554a0513 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -135,13 +135,23 @@ Console I/O .. function:: putch(char) - Print the byte string *char* to the console without buffering. + Print the byte string *char* to the console without buffering. Raises + :exc:`OSError` on failure, for example when the process has no console + attached. + + .. versionchanged:: next + Failures are now reported by raising :exc:`OSError` instead of being + silently ignored. .. function:: putwch(unicode_char) Wide char variant of :func:`putch`, accepting a Unicode value. + .. versionchanged:: next + Failures are now reported by raising :exc:`OSError` instead of being + silently ignored. + .. function:: ungetch(char) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 5858d9733c57796..be7ea26356bebe3 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1642,6 +1642,7 @@ or `the MSDN `_ on Windo - :data:`RWF_APPEND` - :data:`RWF_DONTCACHE` - :data:`RWF_ATOMIC` + - :data:`RWF_NOSIGNAL` Return the total number of bytes actually written. @@ -1691,6 +1692,16 @@ or `the MSDN `_ on Windo .. versionadded:: 3.10 +.. data:: RWF_NOSIGNAL + + Prevent pipe and socket writes from raising :const:`~signal.SIGPIPE`. + This flag is meaningful only for :func:`os.pwritev`. + + .. availability:: Linux >= 6.18. + + .. versionadded:: 3.16 + + .. function:: read(fd, n, /) Read at most *n* bytes from file descriptor *fd*. diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 3b5b57fb1c21702..842e8c512def139 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -137,7 +137,7 @@ Other functions :term:`bytes-like objects ` *a* and *b* are equal, otherwise ``False``, using a "constant-time compare" to reduce the risk of - `timing attacks `_. + `timing attacks `__. See :func:`hmac.compare_digest` for additional details. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 9df5eb78d286a52..0b6a1b488fb64f9 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4876,8 +4876,9 @@ copying. Cast a memoryview to a new format or shape. *shape* defaults to ``[byte_length//new_itemsize]``, which means that the result view will be one-dimensional. The return value is a new memoryview, but - the buffer itself is not copied. Supported casts are 1D -> C-:term:`contiguous` - and C-contiguous -> 1D. + the buffer itself is not copied. Supported casts are + 1D -> C-:term:`contiguous`, C-contiguous -> 1D, and + F-contiguous -> 1D. The destination format is restricted to a single element native format in :mod:`struct` syntax. One of the formats must be a byte format @@ -4964,6 +4965,10 @@ copying. .. versionchanged:: 3.5 The source format is no longer restricted when casting to a byte view. + .. versionchanged:: next + Casting a multi-dimensional F-contiguous view to a one-dimensional + view is now supported. + .. method:: count(value, /) Count the number of occurrences of *value*. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 9a41e406c8c10e6..a2668a38c6b4a2f 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1790,13 +1790,13 @@ always available. Unless explicitly noted otherwise, all variables are read-only callable or ``None`` to clear the filter. The filter function is called for every potentially lazy import to - determine whether it should actually be lazy. It must have the following + determine whether it should actually be lazy. It should have the following signature:: def filter(importing_module: str, imported_module: str, fromlist: tuple[str, ...] | None) -> bool - Where: + The function is called with three positional arguments: * *importing_module* is the name of the module doing the import * *imported_module* is the resolved name of the module being imported @@ -2135,7 +2135,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only returned by the :func:`open` function. Their parameters are chosen as follows: - * The encoding and error handling are is initialized from + * The encoding and error handling are initialized from :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors`. On Windows, UTF-8 is used for the console device. Non-character diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 4e21e1ded82724c..660847ae3fe3c85 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -961,6 +961,59 @@ The :mod:`!test.support` module defines the following functions: :mod:`tracemalloc` is enabled. +.. currentmodule:: test.support.isolation + +.. decorator:: runInSubprocess() + + Decorator that runs the decorated test in a fresh interpreter subprocess, in + isolation, so that it does not share global or interpreter state with the + rest of the test run. It can decorate a test method or a whole + :class:`~unittest.TestCase` subclass. Decorated methods must take no extra + arguments. A failure, error or skip in the subprocess is reported for the + corresponding test, and individual :meth:`subtests + ` that fail or are skipped are reported + individually. A reported failure or error shows the original subprocess + traceback as the cause of the exception. + + When a **method** is decorated, only that method runs in a subprocess; all + fixtures (:meth:`~unittest.TestCase.setUp` / :meth:`~unittest.TestCase.tearDown`, + :meth:`~unittest.TestCase.setUpClass` / :meth:`~unittest.TestCase.tearDownClass` + and ``setUpModule()`` / ``tearDownModule()``) run both in the parent process + (as usual) and in the subprocess around the method. + + When a **class** is decorated, the whole class runs in a single subprocess, + and :meth:`~unittest.TestCase.setUpClass`, + :meth:`~unittest.TestCase.tearDownClass`, :meth:`~unittest.TestCase.setUp` + and :meth:`~unittest.TestCase.tearDown` run once each in the subprocess and + are skipped in the parent process. A failure or skip of + :meth:`~unittest.TestCase.setUpClass` in the subprocess is reported for the + whole class. ``setUpModule()`` cannot be controlled by a class decorator, + so it still runs in the parent process too; test it with + :data:`runningInSubprocess` if needed. + + The subprocess inherits the enabled resources (``-u``), memory limit + (``-M``) and verbosity (``-v``) of the parent test run, so that + :func:`~test.support.requires_resource`, :func:`~test.support.requires`, + :func:`~test.support.bigmemtest` and the like behave consistently in both + processes. + + The test is skipped on platforms without subprocess support. + + +.. data:: runningInSubprocess + + ``True`` while the code runs in the isolated subprocess spawned by + :func:`runInSubprocess`, and ``False`` otherwise (including in the parent + process and in a normal, non-isolated test run). Fixtures such as + :meth:`~unittest.TestCase.setUp`, :meth:`~unittest.TestCase.tearDown`, + :meth:`~unittest.TestCase.setUpClass`, :meth:`~unittest.TestCase.tearDownClass`, + ``setUpModule()`` and ``tearDownModule()`` can test it to choose which code + to run in the subprocess. + + +.. currentmodule:: test.support + + .. function:: check_free_after_iterating(test, iter, cls, args=()) Assert instances of *cls* are deallocated after iterating. diff --git a/Doc/library/uu.rst b/Doc/library/uu.rst index 0636d180294d474..cf6ec0fcef386d2 100644 --- a/Doc/library/uu.rst +++ b/Doc/library/uu.rst @@ -11,5 +11,9 @@ This module is no longer part of the Python standard library. It was :ref:`removed in Python 3.13 ` after being deprecated in Python 3.11. The removal was decided in :pep:`594`. +Encoding and decoding in the uu format can instead be achieved using +:func:`codecs.encode` and :func:`codecs.decode`, specifying ``"uu"`` +as the encoding. + The last version of Python that provided the :mod:`!uu` module was `Python 3.12 `_. diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 9063bea96ccb0ac..d5f89102d5853bb 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -566,7 +566,7 @@ Available Functions and calls to :func:`simplefilter`. -.. decorator:: deprecated(msg, *, category=DeprecationWarning, stacklevel=1) +.. decorator:: deprecated(message, /, *, category=DeprecationWarning, stacklevel=1) Decorator to indicate that a class, function or overload is deprecated. diff --git a/Doc/license.rst b/Doc/license.rst index 23860815e7c1079..47c4fa9d582de3a 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -1216,3 +1216,52 @@ license:: LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Unicode Character Database +-------------------------- + +An extract of the `Unicode Character Database `__, +converted to an internal format, is used by the :mod:`unicodedata` module and +for the Unicode support of the :class:`str` type. The original Unicode data +files are distributed under the `Unicode License `__:: + + UNICODE LICENSE V3 + + COPYRIGHT AND PERMISSION NOTICE + + Copyright © 1991-2026 Unicode, Inc. + + NOTICE TO USER: Carefully read the following legal agreement. BY + DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR + SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE + TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT + DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + + Permission is hereby granted, free of charge, to any person obtaining a + copy of data files and any associated documentation (the "Data Files") or + software and any associated documentation (the "Software") to deal in the + Data Files or Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, and/or sell + copies of the Data Files or Software, and to permit persons to whom the + Data Files or Software are furnished to do so, provided that either (a) + this copyright and permission notice appear with all copies of the Data + Files or Software, or (b) this copyright and permission notice appear in + associated Documentation. + + THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY + KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF + THIRD PARTY RIGHTS. + + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE + BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, + OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, + ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA + FILES OR SOFTWARE. + + Except as contained in this notice, the name of a copyright holder shall + not be used in advertising or otherwise to promote the sale, use or other + dealings in these Data Files or Software without prior written + authorization of the copyright holder. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index c5d474895966119..ab592cfa5a1bbbd 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -12,7 +12,6 @@ Doc/library/email.parser.rst Doc/library/importlib.rst Doc/library/logging.config.rst Doc/library/logging.handlers.rst -Doc/library/lzma.rst Doc/library/mmap.rst Doc/library/multiprocessing.rst Doc/library/optparse.rst diff --git a/Doc/tools/removed-ids.txt b/Doc/tools/removed-ids.txt index 48c9809f0502bfa..7796d15f4d2735f 100644 --- a/Doc/tools/removed-ids.txt +++ b/Doc/tools/removed-ids.txt @@ -43,3 +43,19 @@ c-api/typeobj.html: c.Py_tp_bases # Removed pages library/asyncio-policy.html: (page missing) + +# Removed old Windows installer docs +using/windows.html: diagnostics +using/windows.html: dry-run +using/windows.html: from-a-script +using/windows.html: from-file-associations +using/windows.html: from-the-command-line +using/windows.html: install-on-demand +using/windows.html: install-quiet-option +using/windows.html: installing-without-ui +using/windows.html: modifying-an-install +using/windows.html: python-launcher-for-windows-deprecated +using/windows.html: return-codes +using/windows.html: the-full-installer-deprecated +using/windows.html: virtual-environments +using/windows.html: windows-full diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 22a10db976c4fc6..fd20ff2cb898107 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -1087,6 +1087,32 @@ Libraries options .. versionadded:: 3.10 +.. option:: --with-curses=ncursesw|ncurses|curses + + Designate a backend library for the :mod:`curses` and :mod:`curses.panel` + modules. + + * ``ncursesw``: Use the wide-character ``libncursesw`` (and ``libpanelw``), + found via ``pkg-config``. + * ``ncurses``: Use ``libncurses`` (and ``libpanel``), found via + ``pkg-config``. + * ``curses``: Use the system's native ``libcurses`` (and ``libpanel``), such + as on NetBSD or Solaris. It is built with wide-character support when the + library provides it. + + Without this option (or with ``--with-curses=auto``, the default), Python + tries ``ncursesw`` and then ``ncurses``; the native ``curses`` is used only + when requested explicitly. + + .. versionadded:: next + +.. option:: --without-curses + + Don't build the :mod:`curses` and :mod:`curses.panel` modules, even when a + curses library is present (built by default when one is found). + + .. versionadded:: next + .. option:: --with-libm=STRING Override ``libm`` math library to *STRING* (default is system-dependent). diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 6cf945de5b3f3b4..a01dabbf2475aa4 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -29,8 +29,8 @@ Installation steps For `current Python versions `_ (other than those in ``security`` status), the release team produces a **Python for macOS** installer package for each new release. -A list of available installers -is available `here `_. +A list of available installers is provided on the +`macOS downloads page `__. We recommend using the most recent supported Python version where possible. Current installers provide a `universal2 binary `_ build @@ -39,14 +39,14 @@ supported by a wide range of macOS versions, currently typically from at least **macOS 10.15 Catalina** on. The downloaded file is a standard macOS installer package file (``.pkg``). -File integrity information (checksum, size, sigstore signature, etc) for each file is included +File integrity information (such as checksum, size, and Sigstore signature) for each file is included on the release download page. Installer packages and their contents are signed and notarized with ``Python Software Foundation`` Apple Developer ID certificates to meet `macOS Gatekeeper requirements `_. For a default installation, double-click on the downloaded installer package file. This should launch the standard macOS Installer app and display the first of several -installer windows steps. +installer steps. .. image:: mac_installer_01_introduction.png @@ -73,8 +73,8 @@ display. For most uses, the standard set of installation operations is appropria By pressing the **Customize** button, you can choose to omit or select certain package components of the installer. Click on each package name to see a description of what it installs. -To also install support for the optional free-threaded feature, -see :ref:`install-freethreaded-macos`. +For details of the optional free-threaded feature, which is installed +by default, see :ref:`install-freethreaded-macos`. .. image:: mac_installer_05_custom_install.png @@ -98,7 +98,7 @@ for its use. .. image:: mac_installer_08_install_certificates.png -If ``Successfully installed certifi`` and ``update complete`` appears +If ``Successfully installed certifi`` and ``update complete`` appear in the terminal window, the installation is complete. Close this terminal window and the installer window. @@ -171,7 +171,7 @@ be certain of what you are about to run. .. _alternative_bundles: -Alternative Distributions +Alternative distributions ========================= Besides the standard ``python.org`` for macOS installer, there are third-party @@ -199,7 +199,7 @@ other libraries, and are not maintained or supported by the core Python team. .. _mac-package-manager: -Installing Additional Python Packages +Installing additional Python packages ===================================== Refer to the `Python Packaging User Guide`_ for more information. @@ -211,7 +211,7 @@ Refer to the `Python Packaging User Guide`_ for more information. .. _gui-programming-on-the-mac: -GUI Programming +GUI programming =============== There are several options for building GUI applications on the Mac with Python. @@ -241,19 +241,22 @@ A number of alternative macOS GUI toolkits are available including: supports desktop operating systems. -Advanced Topics +Advanced topics =============== .. _install-freethreaded-macos: -Installing Free-threaded Binaries +Installing free-threaded binaries --------------------------------- .. versionadded:: 3.13 +.. versionchanged:: 3.15 + Free-threading support is now installed by default. + The ``python.org`` :ref:`Python for macOS ` -installer package can optionally install an additional build of -Python |version| that supports :pep:`703`, the free-threading feature +installer package installs, by default, an additional build of +Python |version| that supports :term:`free threading` (running with the :term:`global interpreter lock` disabled). Check the release page on ``python.org`` for possible updated information. @@ -262,14 +265,15 @@ there is some additional overhead in single-threaded workloads compared to the regular build. Additionally, third-party packages, in particular ones with an :term:`extension module`, may not be ready for use in a free-threaded build, and will re-enable the :term:`GIL`. -Therefore, the support for free-threading -is not installed by default. It is packaged as a separate install option, -available by clicking the **Customize** button on the **Installation Type** +As of Python 3.15, free-threading support is installed by default. +It is packaged as a separate install option, which can be deselected +by clicking the **Customize** button on the **Installation Type** step of the installer as described above. .. image:: mac_installer_09_custom_install_free_threaded.png -If the box next to the **Free-threaded Python** package name is checked, +If the box next to the **Free-threaded Python** package name is checked +(as it is by default), a separate :file:`PythonT.framework` will also be installed alongside the normal :file:`Python.framework` in :file:`/Library/Frameworks`. This configuration allows a free-threaded Python |version| build to co-exist @@ -284,7 +288,7 @@ Known cautions and limitations: the free-threaded interpreter, and |python_x_dot_y_t_literal_config|, a configuration utility which may be useful for package builders. Since :file:`/usr/local/bin` is typically included in your shell ``PATH``, - in most cases no changes to your ``PATH`` environment variables should + in most cases no changes to your ``PATH`` environment variable should be needed to use |python_x_dot_y_t_literal|. - For this release, the **Shell profile updater** package and the @@ -367,8 +371,8 @@ the macOS command line :command:`installer` utility lets you select non-default options, too. If you are not familiar with :command:`installer`, it can be somewhat cryptic (see :command:`man installer` for more information). As an example, the following shell snippet shows one way to do it, -using the |x_dot_y_b2_literal| release and selecting the free-threaded interpreter -option: +using the |x_dot_y_b2_literal| release and deselecting the free-threaded +interpreter option: .. parsed-literal:: @@ -378,7 +382,7 @@ option: curl -O \https://www.python.org/ftp/python/\ |version|\ .0/${RELEASE} # create installer choicechanges to customize the install: - # enable the PythonTFramework-\ |version|\ package + # disable the PythonTFramework-\ |version|\ package # while accepting the other defaults (install all other packages) cat > ./choicechanges.plist < @@ -387,7 +391,7 @@ option: attributeSetting - 1 + 0 choiceAttribute selected choiceIdentifier @@ -400,7 +404,7 @@ option: sudo installer -pkg ./${RELEASE} -applyChoiceChangesXML ./choicechanges.plist -target / -You can then test that both installer builds are now available with something like: +After a default install, you can test that both builds are available with something like: .. parsed-literal:: @@ -425,7 +429,7 @@ You can then test that both installer builds are now available with something li .. _distributing-python-applications-on-the-mac: -Distributing Python Applications +Distributing Python applications -------------------------------- A range of tools exist for converting your Python code into a standalone @@ -442,7 +446,7 @@ distributable application: * `PyInstaller `_: A cross-platform packaging tool that creates a single file or folder as a distributable artifact. -App Store Compliance +App Store compliance -------------------- Apps submitted for distribution through the macOS App Store must pass Apple's @@ -464,7 +468,7 @@ This patch is not normally required to use CPython on a Mac; nor is it required if you are distributing an app *outside* the macOS App Store. It is *only* required if you are using the macOS App Store as a distribution channel. -Other Resources +Other resources =============== The `python.org Help page `_ has links to many useful resources. diff --git a/Doc/using/mac_installer_01_introduction.png b/Doc/using/mac_installer_01_introduction.png index 1999f3a37590933..81e1f4e5ac5c32e 100644 Binary files a/Doc/using/mac_installer_01_introduction.png and b/Doc/using/mac_installer_01_introduction.png differ diff --git a/Doc/using/mac_installer_02_readme.png b/Doc/using/mac_installer_02_readme.png index a36efaf7d50fd6a..7e4a9b15cf94f6a 100644 Binary files a/Doc/using/mac_installer_02_readme.png and b/Doc/using/mac_installer_02_readme.png differ diff --git a/Doc/using/mac_installer_03_license.png b/Doc/using/mac_installer_03_license.png index 598c22a13d9e62e..153daa780efa9e9 100644 Binary files a/Doc/using/mac_installer_03_license.png and b/Doc/using/mac_installer_03_license.png differ diff --git a/Doc/using/mac_installer_04_installation_type.png b/Doc/using/mac_installer_04_installation_type.png index 9498fd06240a4ed..1fc597c2427ab76 100644 Binary files a/Doc/using/mac_installer_04_installation_type.png and b/Doc/using/mac_installer_04_installation_type.png differ diff --git a/Doc/using/mac_installer_05_custom_install.png b/Doc/using/mac_installer_05_custom_install.png index 3a201d2f44655af..c70d93bbcee70a8 100644 Binary files a/Doc/using/mac_installer_05_custom_install.png and b/Doc/using/mac_installer_05_custom_install.png differ diff --git a/Doc/using/mac_installer_06_summary.png b/Doc/using/mac_installer_06_summary.png index 1af6eee2c668cdb..f8a67052ef04479 100644 Binary files a/Doc/using/mac_installer_06_summary.png and b/Doc/using/mac_installer_06_summary.png differ diff --git a/Doc/using/mac_installer_07_applications.png b/Doc/using/mac_installer_07_applications.png index c8b3aa1099adfc9..c39741f33ac807f 100644 Binary files a/Doc/using/mac_installer_07_applications.png and b/Doc/using/mac_installer_07_applications.png differ diff --git a/Doc/using/mac_installer_08_install_certificates.png b/Doc/using/mac_installer_08_install_certificates.png index c125eeb18aa0c14..f476c7f37d245d6 100644 Binary files a/Doc/using/mac_installer_08_install_certificates.png and b/Doc/using/mac_installer_08_install_certificates.png differ diff --git a/Doc/using/mac_installer_09_custom_install_free_threaded.png b/Doc/using/mac_installer_09_custom_install_free_threaded.png index 0f69c55eddb228b..e867e422e42e485 100644 Binary files a/Doc/using/mac_installer_09_custom_install_free_threaded.png and b/Doc/using/mac_installer_09_custom_install_free_threaded.png differ diff --git a/Doc/using/win_install_freethreaded.png b/Doc/using/win_install_freethreaded.png deleted file mode 100644 index 12b89c0165d994d..000000000000000 Binary files a/Doc/using/win_install_freethreaded.png and /dev/null differ diff --git a/Doc/using/win_installer.png b/Doc/using/win_installer.png deleted file mode 100644 index fc9605a79cfcc51..000000000000000 Binary files a/Doc/using/win_installer.png and /dev/null differ diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index e48de587598e5fe..40854332af0b24f 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -59,6 +59,8 @@ packages. .. _setting-envvars: .. _windows-path-mod: .. _launcher: +.. _getting-started: +.. _installation-steps: Python install manager ====================== @@ -128,7 +130,7 @@ difference between the two commands is when running without any arguments: help (``pymanager exec ...`` provides equivalent behaviour to ``py ...``). Each of these commands also has a windowed version that avoids creating a -console window. These are ``pyw``, ``pythonw`` and ``pymanagerw``. A ``python3`` +console window. These are ``pyw``, ``pythonw`` and ``pywmanager``. A ``python3`` command is also included that mimics the ``python`` command. It is intended to catch accidental uses of the typical POSIX command on Windows, but is not meant to be widely used or recommended. @@ -315,6 +317,8 @@ for new users, but should not generally be relied on rather than using the .. _pymanager-offline: +.. _install-layout-option: +.. _installing-without-downloading: Offline installs ---------------- @@ -378,6 +382,10 @@ again. To completely clean up all Python runtimes, run with ``--purge`` before uninstalling the Python install manager. .. _pymanager-config: +.. _customization: +.. _customizing-default-python-versions: +.. _customization-via-ini-files: +.. _launcher-ini: Configuration ------------- @@ -518,6 +526,7 @@ Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. .. _pymanager-shebang: +.. _arguments-in-shebang-lines: Shebang lines ------------- @@ -1501,725 +1510,3 @@ releases. These files are in the :file:`PCbuild` directory. Check :file:`PCbuild/readme.txt` for general information on the build process. For extension modules, consult :ref:`building-on-windows`. - - - -.. _windows-full: - -The full installer (deprecated) -=============================== - -.. deprecated:: 3.14 - - This installer is deprecated since 3.14 and will not be produced for Python - 3.16 or later. See :ref:`pymanager` for the modern installer. - - -Installation steps ------------------- - -Four Python |version| installers are available for download - two each for the -32-bit and 64-bit versions of the interpreter. The *web installer* is a small -initial download, and it will automatically download the required components as -necessary. The *offline installer* includes the components necessary for a -default installation and only requires an internet connection for optional -features. See :ref:`install-layout-option` for other ways to avoid downloading -during installation. - -After starting the installer, one of two options may be selected: - -.. image:: win_installer.png - -If you select "Install Now": - -* You will *not* need to be an administrator (unless a system update for the - C Runtime Library is required or you install the :ref:`launcher` for all - users) -* Python will be installed into your user directory -* The :ref:`launcher` will be installed according to the option at the bottom - of the first page -* The standard library, test suite, launcher and pip will be installed -* If selected, the install directory will be added to your :envvar:`PATH` -* Shortcuts will only be visible for the current user - -Selecting "Customize installation" will allow you to select the features to -install, the installation location and other options or post-install actions. -To install debugging symbols or binaries, you will need to use this option. - -To perform an all-users installation, you should select "Customize -installation". In this case: - -* You may be required to provide administrative credentials or approval -* Python will be installed into the Program Files directory -* The :ref:`launcher` will be installed into the Windows directory -* Optional features may be selected during installation -* The standard library can be pre-compiled to bytecode -* If selected, the install directory will be added to the system :envvar:`PATH` -* Shortcuts are available for all users - - -Removing the MAX_PATH limitation --------------------------------- - -Windows historically has limited path lengths to 260 characters. This meant that -paths longer than this would not resolve and errors would result. - -In the latest versions of Windows, this limitation can be expanded to -approximately 32,000 characters. Your administrator will need to activate the -"Enable Win32 long paths" group policy, or set ``LongPathsEnabled`` to ``1`` -in the registry key -``HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem``. - -This allows the :func:`open` function, the :mod:`os` module and most other -path functionality to accept and return paths longer than 260 characters. - -After changing the above option, no further configuration is required. - -.. versionchanged:: 3.6 - - Support for long paths was enabled in Python. - -.. _install-quiet-option: - -Installing without UI ---------------------- - -All of the options available in the installer UI can also be specified from the -command line, allowing scripted installers to replicate an installation on many -machines without user interaction. These options may also be set without -suppressing the UI in order to change some of the defaults. - -The following options (found by executing the installer with ``/?``) can be -passed into the installer: - -+---------------------+--------------------------------------------------------+ -| Name | Description | -+=====================+========================================================+ -| /passive | to display progress without requiring user interaction | -+---------------------+--------------------------------------------------------+ -| /quiet | to install/uninstall without displaying any UI | -+---------------------+--------------------------------------------------------+ -| /simple | to prevent user customization | -+---------------------+--------------------------------------------------------+ -| /uninstall | to remove Python (without confirmation) | -+---------------------+--------------------------------------------------------+ -| /layout [directory] | to pre-download all components | -+---------------------+--------------------------------------------------------+ -| /log [filename] | to specify log files location | -+---------------------+--------------------------------------------------------+ - -All other options are passed as ``name=value``, where the value is usually -``0`` to disable a feature, ``1`` to enable a feature, or a path. The full list -of available options is shown below. - -+---------------------------+--------------------------------------+--------------------------+ -| Name | Description | Default | -+===========================+======================================+==========================+ -| InstallAllUsers | Perform a system-wide installation. | 0 | -+---------------------------+--------------------------------------+--------------------------+ -| TargetDir | The installation directory | Selected based on | -| | | InstallAllUsers | -+---------------------------+--------------------------------------+--------------------------+ -| DefaultAllUsersTargetDir | The default installation directory | :file:`%ProgramFiles%\\\ | -| | for all-user installs | Python X.Y` or :file:`\ | -| | | %ProgramFiles(x86)%\\\ | -| | | Python X.Y` | -+---------------------------+--------------------------------------+--------------------------+ -| DefaultJustForMeTargetDir | The default install directory for | :file:`%LocalAppData%\\\ | -| | just-for-me installs | Programs\\Python\\\ | -| | | PythonXY` or | -| | | :file:`%LocalAppData%\\\ | -| | | Programs\\Python\\\ | -| | | PythonXY-32` or | -| | | :file:`%LocalAppData%\\\ | -| | | Programs\\Python\\\ | -| | | PythonXY-64` | -+---------------------------+--------------------------------------+--------------------------+ -| DefaultCustomTargetDir | The default custom install directory | (empty) | -| | displayed in the UI | | -+---------------------------+--------------------------------------+--------------------------+ -| AssociateFiles | Create file associations if the | 1 | -| | launcher is also installed. | | -+---------------------------+--------------------------------------+--------------------------+ -| CompileAll | Compile all ``.py`` files to | 0 | -| | ``.pyc``. | | -+---------------------------+--------------------------------------+--------------------------+ -| PrependPath | Prepend install and Scripts | 0 | -| | directories to :envvar:`PATH` and | | -| | add ``.PY`` to :envvar:`PATHEXT` | | -+---------------------------+--------------------------------------+--------------------------+ -| AppendPath | Append install and Scripts | 0 | -| | directories to :envvar:`PATH` and | | -| | add ``.PY`` to :envvar:`PATHEXT` | | -+---------------------------+--------------------------------------+--------------------------+ -| Shortcuts | Create shortcuts for the interpreter,| 1 | -| | documentation and IDLE if installed. | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_doc | Install Python manual | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_debug | Install debug binaries | 0 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_dev | Install developer headers and | 1 | -| | libraries. Omitting this may lead to | | -| | an unusable installation. | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_exe | Install :file:`python.exe` and | 1 | -| | related files. Omitting this may | | -| | lead to an unusable installation. | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_launcher | Install :ref:`launcher`. | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| InstallLauncherAllUsers | Installs the launcher for all | 1 | -| | users. Also requires | | -| | ``Include_launcher`` to be set to 1 | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_lib | Install standard library and | 1 | -| | extension modules. Omitting this may | | -| | lead to an unusable installation. | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_pip | Install bundled pip and setuptools | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_symbols | Install debugging symbols (``*.pdb``)| 0 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_tcltk | Install Tcl/Tk support and IDLE | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_test | Install standard library test suite | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_tools | Install utility scripts | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| LauncherOnly | Only installs the launcher. This | 0 | -| | will override most other options. | | -+---------------------------+--------------------------------------+--------------------------+ -| SimpleInstall | Disable most install UI | 0 | -+---------------------------+--------------------------------------+--------------------------+ -| SimpleInstallDescription | A custom message to display when the | (empty) | -| | simplified install UI is used. | | -+---------------------------+--------------------------------------+--------------------------+ - -For example, to silently install a default, system-wide Python installation, -you could use the following command (from an elevated command prompt):: - - python-3.9.0.exe /quiet InstallAllUsers=1 PrependPath=1 Include_test=0 - -To allow users to easily install a personal copy of Python without the test -suite, you could provide a shortcut with the following command. This will -display a simplified initial page and disallow customization:: - - python-3.9.0.exe InstallAllUsers=0 Include_launcher=0 Include_test=0 - SimpleInstall=1 SimpleInstallDescription="Just for me, no test suite." - -(Note that omitting the launcher also omits file associations, and is only -recommended for per-user installs when there is also a system-wide installation -that included the launcher.) - -The options listed above can also be provided in a file named ``unattend.xml`` -alongside the executable. This file specifies a list of options and values. -When a value is provided as an attribute, it will be converted to a number if -possible. Values provided as element text are always left as strings. This -example file sets the same options as the previous example: - -.. code-block:: xml - - - - - -.. _install-layout-option: - -Installing without downloading ------------------------------- - -As some features of Python are not included in the initial installer download, -selecting those features may require an internet connection. To avoid this -need, all possible components may be downloaded on-demand to create a complete -*layout* that will no longer require an internet connection regardless of the -selected features. Note that this download may be bigger than required, but -where a large number of installations are going to be performed it is very -useful to have a locally cached copy. - -Execute the following command from Command Prompt to download all possible -required files. Remember to substitute ``python-3.9.0.exe`` for the actual -name of your installer, and to create layouts in their own directories to -avoid collisions between files with the same name. - -:: - - python-3.9.0.exe /layout [optional target directory] - -You may also specify the ``/quiet`` option to hide the progress display. - -Modifying an install --------------------- - -Once Python has been installed, you can add or remove features through the -Programs and Features tool that is part of Windows. Select the Python entry and -choose "Uninstall/Change" to open the installer in maintenance mode. - -"Modify" allows you to add or remove features by modifying the checkboxes - -unchanged checkboxes will not install or remove anything. Some options cannot be -changed in this mode, such as the install directory; to modify these, you will -need to remove and then reinstall Python completely. - -"Repair" will verify all the files that should be installed using the current -settings and replace any that have been removed or modified. - -"Uninstall" will remove Python entirely, with the exception of the -:ref:`launcher`, which has its own entry in Programs and Features. - - -Installing free-threaded binaries ---------------------------------- - -.. versionadded:: 3.13 - -To install pre-built binaries with free-threading enabled (see :pep:`703`), you -should select "Customize installation". The second page of options includes the -"Download free-threaded binaries" checkbox. - -.. image:: win_install_freethreaded.png - -Selecting this option will download and install additional binaries to the same -location as the main Python install. The main executable is called -``python3.13t.exe``, and other binaries either receive a ``t`` suffix or a full -ABI suffix. Python source files and bundled third-party dependencies are shared -with the main install. - -The free-threaded version is registered as a regular Python install with the -tag ``3.13t`` (with a ``-32`` or ``-arm64`` suffix as normal for those -platforms). This allows tools to discover it, and for the :ref:`launcher` to -support ``py.exe -3.13t``. Note that the launcher will interpret ``py.exe -3`` -(or a ``python3`` shebang) as "the latest 3.x install", which will prefer the -free-threaded binaries over the regular ones, while ``py.exe -3.13`` will not. -If you use the short style of option, you may prefer to not install the -free-threaded binaries at this time. - -To specify the install option at the command line, use -``Include_freethreaded=1``. See :ref:`install-layout-option` for instructions on -pre-emptively downloading the additional binaries for offline install. The -options to include debug symbols and binaries also apply to the free-threaded -builds. - -Free-threaded binaries are also available :ref:`on nuget.org `. - - -Python launcher for Windows (deprecated) -======================================== - -.. deprecated:: 3.14 - - The launcher and this documentation have been superseded by the Python - Install Manager described above. This is preserved temporarily for historical - interest. - -.. versionadded:: 3.3 - -The Python launcher for Windows is a utility which aids in locating and -executing of different Python versions. It allows scripts (or the -command-line) to indicate a preference for a specific Python version, and -will locate and execute that version. - -Unlike the :envvar:`PATH` variable, the launcher will correctly select the most -appropriate version of Python. It will prefer per-user installations over -system-wide ones, and orders by language version rather than using the most -recently installed version. - -The launcher was originally specified in :pep:`397`. - -Getting started ---------------- - -From the command-line -^^^^^^^^^^^^^^^^^^^^^ - -.. versionchanged:: 3.6 - -System-wide installations of Python 3.3 and later will put the launcher on your -:envvar:`PATH`. The launcher is compatible with all available versions of -Python, so it does not matter which version is installed. To check that the -launcher is available, execute the following command in Command Prompt:: - - py - -You should find that the latest version of Python you have installed is -started - it can be exited as normal, and any additional command-line -arguments specified will be sent directly to Python. - -If you have multiple versions of Python installed (e.g., 3.7 and |version|) you -will have noticed that Python |version| was started - to launch Python 3.7, try -the command:: - - py -3.7 - -If you want the latest version of Python 2 you have installed, try the -command:: - - py -2 - -If you see the following error, you do not have the launcher installed:: - - 'py' is not recognized as an internal or external command, - operable program or batch file. - -The command:: - - py --list - -displays the currently installed version(s) of Python. - -The ``-x.y`` argument is the short form of the ``-V:Company/Tag`` argument, -which allows selecting a specific Python runtime, including those that may have -come from somewhere other than python.org. Any runtime registered by following -:pep:`514` will be discoverable. The ``--list`` command lists all available -runtimes using the ``-V:`` format. - -When using the ``-V:`` argument, specifying the Company will limit selection to -runtimes from that provider, while specifying only the Tag will select from all -providers. Note that omitting the slash implies a tag:: - - # Select any '3.*' tagged runtime - py -V:3 - - # Select any 'PythonCore' released runtime - py -V:PythonCore/ - - # Select PythonCore's latest Python 3 runtime - py -V:PythonCore/3 - -The short form of the argument (``-3``) only ever selects from core Python -releases, and not other distributions. However, the longer form (``-V:3``) will -select from any. - -The Company is matched on the full string, case-insensitive. The Tag is matched -on either the full string, or a prefix, provided the next character is a dot or a -hyphen. This allows ``-V:3.1`` to match ``3.1-32``, but not ``3.10``. Tags are -sorted using numerical ordering (``3.10`` is newer than ``3.1``), but are -compared using text (``-V:3.01`` does not match ``3.1``). - - -Virtual environments -^^^^^^^^^^^^^^^^^^^^ - -.. versionadded:: 3.5 - -If the launcher is run with no explicit Python version specification, and a -virtual environment (created with the standard library :mod:`venv` module or -the external ``virtualenv`` tool) active, the launcher will run the virtual -environment's interpreter rather than the global one. To run the global -interpreter, either deactivate the virtual environment, or explicitly specify -the global Python version. - -From a script -^^^^^^^^^^^^^ - -Let's create a test Python script - create a file called ``hello.py`` with the -following contents - -.. code-block:: python - - #! python - import sys - sys.stdout.write("hello from Python %s\n" % (sys.version,)) - -From the directory in which hello.py lives, execute the command:: - - py hello.py - -You should notice the version number of your latest Python 2.x installation -is printed. Now try changing the first line to be: - -.. code-block:: python - - #! python3 - -Re-executing the command should now print the latest Python 3.x information. -As with the above command-line examples, you can specify a more explicit -version qualifier. Assuming you have Python 3.7 installed, try changing -the first line to ``#! python3.7`` and you should find the 3.7 -version information printed. - -Note that unlike interactive use, a bare "python" will use the latest -version of Python 2.x that you have installed. This is for backward -compatibility and for compatibility with Unix, where the command ``python`` -typically refers to Python 2. - -From file associations -^^^^^^^^^^^^^^^^^^^^^^ - -The launcher should have been associated with Python files (i.e. ``.py``, -``.pyw``, ``.pyc`` files) when it was installed. This means that -when you double-click on one of these files from Windows explorer the launcher -will be used, and therefore you can use the same facilities described above to -have the script specify the version which should be used. - -The key benefit of this is that a single launcher can support multiple Python -versions at the same time depending on the contents of the first line. - -Shebang lines -------------- - -If the first line of a script file starts with ``#!``, it is known as a -"shebang" line. Linux and other Unix like operating systems have native -support for such lines and they are commonly used on such systems to indicate -how a script should be executed. This launcher allows the same facilities to -be used with Python scripts on Windows and the examples above demonstrate their -use. - -To allow shebang lines in Python scripts to be portable between Unix and -Windows, this launcher supports a number of 'virtual' commands to specify -which interpreter to use. The supported virtual commands are: - -* ``/usr/bin/env`` -* ``/usr/bin/python`` -* ``/usr/local/bin/python`` -* ``python`` - -For example, if the first line of your script starts with - -.. code-block:: sh - - #! /usr/bin/python - -The default Python or an active virtual environment will be located and used. -As many Python scripts written to work on Unix will already have this line, -you should find these scripts can be used by the launcher without modification. -If you are writing a new script on Windows which you hope will be useful on -Unix, you should use one of the shebang lines starting with ``/usr``. - -Any of the above virtual commands can be suffixed with an explicit version -(either just the major version, or the major and minor version). -Furthermore the 32-bit version can be requested by adding "-32" after the -minor version. I.e. ``/usr/bin/python3.7-32`` will request usage of the -32-bit Python 3.7. If a virtual environment is active, the version will be -ignored and the environment will be used. - -.. versionadded:: 3.7 - - Beginning with python launcher 3.7 it is possible to request 64-bit version - by the "-64" suffix. Furthermore it is possible to specify a major and - architecture without minor (i.e. ``/usr/bin/python3-64``). - -.. versionchanged:: 3.11 - - The "-64" suffix is deprecated, and now implies "any architecture that is - not provably i386/32-bit". To request a specific environment, use the new - :samp:`-V:{TAG}` argument with the complete tag. - -.. versionchanged:: 3.13 - - Virtual commands referencing ``python`` now prefer an active virtual - environment rather than searching :envvar:`PATH`. This handles cases where - the shebang specifies ``/usr/bin/env python3`` but :file:`python3.exe` is - not present in the active environment. - -The ``/usr/bin/env`` form of shebang line has one further special property. -Before looking for installed Python interpreters, this form will search the -executable :envvar:`PATH` for a Python executable matching the name provided -as the first argument. This corresponds to the behaviour of the Unix ``env`` -program, which performs a :envvar:`PATH` search. -If an executable matching the first argument after the ``env`` command cannot -be found, but the argument starts with ``python``, it will be handled as -described for the other virtual commands. -The environment variable :envvar:`!PYLAUNCHER_NO_SEARCH_PATH` may be set -(to any value) to skip this search of :envvar:`PATH`. - -Shebang lines that do not match any of these patterns are looked up in the -``[commands]`` section of the launcher's :ref:`.INI file `. -This may be used to handle certain commands in a way that makes sense for your -system. The name of the command must be a single argument (no spaces in the -shebang executable), and the value substituted is the full path to the -executable (additional arguments specified in the .INI will be quoted as part -of the filename). - -.. code-block:: ini - - [commands] - /bin/xpython=C:\Program Files\XPython\python.exe - -Any commands not found in the .INI file are treated as **Windows** executable -paths that are absolute or relative to the directory containing the script file. -This is a convenience for Windows-only scripts, such as those generated by an -installer, since the behavior is not compatible with Unix-style shells. -These paths may be quoted, and may include multiple arguments, after which the -path to the script and any additional arguments will be appended. - - -Arguments in shebang lines --------------------------- - -The shebang lines can also specify additional options to be passed to the -Python interpreter. For example, if you have a shebang line: - -.. code-block:: sh - - #! /usr/bin/python -v - -Then Python will be started with the ``-v`` option - -Customization -------------- - -.. _launcher-ini: - -Customization via INI files -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Two .ini files will be searched by the launcher - ``py.ini`` in the current -user's application data directory (``%LOCALAPPDATA%`` or ``$env:LocalAppData``) -and ``py.ini`` in the same directory as the launcher. The same .ini files are -used for both the 'console' version of the launcher (i.e. py.exe) and for the -'windows' version (i.e. pyw.exe). - -Customization specified in the "application directory" will have precedence over -the one next to the executable, so a user, who may not have write access to the -.ini file next to the launcher, can override commands in that global .ini file. - -Customizing default Python versions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In some cases, a version qualifier can be included in a command to dictate -which version of Python will be used by the command. A version qualifier -starts with a major version number and can optionally be followed by a period -('.') and a minor version specifier. Furthermore it is possible to specify -if a 32 or 64 bit implementation shall be requested by adding "-32" or "-64". - -For example, a shebang line of ``#!python`` has no version qualifier, while -``#!python3`` has a version qualifier which specifies only a major version. - -If no version qualifiers are found in a command, the environment -variable :envvar:`!PY_PYTHON` can be set to specify the default version -qualifier. If it is not set, the default is "3". The variable can -specify any value that may be passed on the command line, such as "3", -"3.7", "3.7-32" or "3.7-64". (Note that the "-64" option is only -available with the launcher included with Python 3.7 or newer.) - -If no minor version qualifiers are found, the environment variable -``PY_PYTHON{major}`` (where ``{major}`` is the current major version qualifier -as determined above) can be set to specify the full version. If no such option -is found, the launcher will enumerate the installed Python versions and use -the latest minor release found for the major version, which is likely, -although not guaranteed, to be the most recently installed version in that -family. - -On 64-bit Windows with both 32-bit and 64-bit implementations of the same -(major.minor) Python version installed, the 64-bit version will always be -preferred. This will be true for both 32-bit and 64-bit implementations of the -launcher - a 32-bit launcher will prefer to execute a 64-bit Python installation -of the specified version if available. This is so the behavior of the launcher -can be predicted knowing only what versions are installed on the PC and -without regard to the order in which they were installed (i.e., without knowing -whether a 32 or 64-bit version of Python and corresponding launcher was -installed last). As noted above, an optional "-32" or "-64" suffix can be -used on a version specifier to change this behaviour. - -Examples: - -* If no relevant options are set, the commands ``python`` and - ``python2`` will use the latest Python 2.x version installed and - the command ``python3`` will use the latest Python 3.x installed. - -* The command ``python3.7`` will not consult any - options at all as the versions are fully specified. - -* If ``PY_PYTHON=3``, the commands ``python`` and ``python3`` will both use - the latest installed Python 3 version. - -* If ``PY_PYTHON=3.7-32``, the command ``python`` will use the 32-bit - implementation of 3.7 whereas the command ``python3`` will use the latest - installed Python (PY_PYTHON was not considered at all as a major - version was specified.) - -* If ``PY_PYTHON=3`` and ``PY_PYTHON3=3.7``, the commands - ``python`` and ``python3`` will both use specifically 3.7 - -In addition to environment variables, the same settings can be configured -in the .INI file used by the launcher. The section in the INI file is -called ``[defaults]`` and the key name will be the same as the -environment variables without the leading ``PY_`` prefix (and note that -the key names in the INI file are case insensitive.) The contents of -an environment variable will override things specified in the INI file. - -For example: - -* Setting ``PY_PYTHON=3.7`` is equivalent to the INI file containing: - -.. code-block:: ini - - [defaults] - python=3.7 - -* Setting ``PY_PYTHON=3`` and ``PY_PYTHON3=3.7`` is equivalent to the INI file - containing: - -.. code-block:: ini - - [defaults] - python=3 - python3=3.7 - -Diagnostics ------------ - -If an environment variable :envvar:`!PYLAUNCHER_DEBUG` is set (to any value), the -launcher will print diagnostic information to stderr (i.e. to the console). -While this information manages to be simultaneously verbose *and* terse, it -should allow you to see what versions of Python were located, why a -particular version was chosen and the exact command-line used to execute the -target Python. It is primarily intended for testing and debugging. - -Dry run -------- - -If an environment variable :envvar:`!PYLAUNCHER_DRYRUN` is set (to any value), -the launcher will output the command it would have run, but will not actually -launch Python. This may be useful for tools that want to use the launcher to -detect and then launch Python directly. Note that the command written to -standard output is always encoded using UTF-8, and may not render correctly in -the console. - -Install on demand ------------------ - -If an environment variable :envvar:`!PYLAUNCHER_ALLOW_INSTALL` is set (to any -value), and the requested Python version is not installed but is available on -the Microsoft Store, the launcher will attempt to install it. This may require -user interaction to complete, and you may need to run the command again. - -An additional :envvar:`!PYLAUNCHER_ALWAYS_INSTALL` variable causes the launcher -to always try to install Python, even if it is detected. This is mainly intended -for testing (and should be used with :envvar:`!PYLAUNCHER_DRYRUN`). - -Return codes ------------- - -The following exit codes may be returned by the Python launcher. Unfortunately, -there is no way to distinguish these from the exit code of Python itself. - -The names of codes are as used in the sources, and are only for reference. There -is no way to access or resolve them apart from reading this page. Entries are -listed in alphabetical order of names. - -+-------------------+-------+-----------------------------------------------+ -| Name | Value | Description | -+===================+=======+===============================================+ -| RC_BAD_VENV_CFG | 107 | A :file:`pyvenv.cfg` was found but is corrupt.| -+-------------------+-------+-----------------------------------------------+ -| RC_CREATE_PROCESS | 101 | Failed to launch Python. | -+-------------------+-------+-----------------------------------------------+ -| RC_INSTALLING | 111 | An install was started, but the command will | -| | | need to be re-run after it completes. | -+-------------------+-------+-----------------------------------------------+ -| RC_INTERNAL_ERROR | 109 | Unexpected error. Please report a bug. | -+-------------------+-------+-----------------------------------------------+ -| RC_NO_COMMANDLINE | 108 | Unable to obtain command line from the | -| | | operating system. | -+-------------------+-------+-----------------------------------------------+ -| RC_NO_PYTHON | 103 | Unable to locate the requested version. | -+-------------------+-------+-----------------------------------------------+ -| RC_NO_VENV_CFG | 106 | A :file:`pyvenv.cfg` was required but not | -| | | found. | -+-------------------+-------+-----------------------------------------------+ - diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 7b3475882063635..4adb5cbd1b7d3b9 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -522,7 +522,7 @@ Other CPython Implementation Changes have been removed, as they can be derived from ``exc_value``. (Contributed by Irit Katriel in :issue:`45711`.) -* A new :ref:`command line option `, ``AppendPath``, +* A new command line option, ``AppendPath``, has been added for the Windows installer. It behaves similarly to ``PrependPath``, but appends the install and scripts directories instead of prepending them. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2fa7267292d9940..64c6888d9947702 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -3312,6 +3312,16 @@ Changes in the Python API This temporary change affects other threads. (Contributed by Serhiy Storchaka in :gh:`69998`.) +* :func:`pickle.dump` and :func:`pickle.dumps` now raise + :exc:`~pickle.PicklingError` for some failures that previously raised + :exc:`AttributeError`, :exc:`ImportError`, :exc:`ValueError`, + :exc:`UnicodeEncodeError` or :exc:`!PicklingError`, + depending on the implementation + (for example, pickling a local object, or an object whose module cannot be imported). + The original exception is chained to the :exc:`!PicklingError`. + Code that caught these exceptions should also catch :exc:`!PicklingError`. + (Contributed by Serhiy Storchaka in :gh:`122311`.) + * :class:`types.UnionType` is now an alias for :class:`typing.Union`, causing changes in some behaviors. See :ref:`above ` for more details. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 8cba187bf31dd16..daac31fd7f4b17c 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -873,6 +873,14 @@ Other language changes imports ``pkg.sub.mod``. (Contributed by Gregory P. Smith in :gh:`83065`.) +* File names of Stable ABI extensions that use the ``.so`` suffix may now + include a multiarch tuple, for example, ``foo.abi3-x86-64-linux-gnu.so``. + This permits stable ABI extensions for multiple architectures to be + co-installed into the same directory, without clashing with each + other, as regular dynamic extensions do. + (Contributed by Stefano Rivera in :gh:`122931`.) + + Default interactive shell ========================= @@ -1225,7 +1233,7 @@ importlib.metadata would return an empty ``PackageMetadata`` object as if the file was present but empty. Now, a ``MetadataNotFound`` exception is raised. See `importlib_metadata#493 `_ - for background and rationale and and :gh:`143387` for rationale on the + for background and rationale and :gh:`143387` for rationale on the compatibility concerns. (Contributed by Jason R. Coombs.) @@ -2309,6 +2317,13 @@ New deprecations :func:`issubclass`, but warnings were not previously emitted if it was merely imported or accessed from the :mod:`!typing` module. + * It is deprecated to call :func:`isinstance` and :func:`issubclass` checks on + protocol classes that were not explicitly decorated with :func:`!runtime_checkable` + but that inherit from a runtime-checkable protocol class. + This will raise a :exc:`TypeError` in Python 3.20. + + (Contributed by Bartosz Sławecki in :gh:`132604`.) + * :mod:`webbrowser`: diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 5cb1a86dac46233..06e831ea1e34631 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -75,6 +75,10 @@ New features Other language changes ====================== +* :meth:`memoryview.cast` now allows casting a multidimensional + F-contiguous view to a one-dimensional view. + (Contributed by Jaemin Park in :gh:`91484`.) + * :ref:`Frame objects ` now support :mod:`weak references `. This allows associating extra data with active frames, for example in debuggers, without keeping the frames (and everything @@ -101,6 +105,21 @@ codecs even when a built-in codec of the same name exists. (Contributed by Serhiy Storchaka in :gh:`152997`.) +csv +--- + +* :meth:`csv.Sniffer.sniff` now deduces the dialect by trial parsing + instead of heuristics based on matching isolated fragments + and on character frequencies, + so the result is consistent with how :func:`csv.reader` will parse the sample. + It can now detect the *escapechar* parameter, + accepts non-ASCII delimiters in the *delimiters* argument, + no longer misdetects the delimiter + when the sample contains delimiter characters inside quoted fields, + and no longer takes quadratic time on quoted samples. + The results may differ from those of earlier Python versions. + (Contributed by Serhiy Storchaka in :gh:`83273`.) + curses ------ @@ -237,6 +256,9 @@ ctypes from an annotation-based syntax, similar to how the :mod:`dataclasses` module is used. (Contributed by Peter Bierma in :gh:`104533`.) +* Add :func:`ctypes.util.wrap_dll_function` for generating function pointers + through a function signature. + (Contributed by Peter Bierma in :gh:`153903`.) encodings @@ -362,6 +384,17 @@ os (Contributed by Maurycy Pawłowski-Wieroński in :gh:`149464`.) +pydoc +----- + +* Modernize the HTML output of :mod:`pydoc`. The pages generated by the + :mod:`pydoc` HTTP server now use semantic HTML5 markup and a new style + sheet based on the python-docs-theme used by `docs.python.org + `__, with dark mode support. Class members + inherited from other classes are collapsed by default. + (Contributed by Serhiy Storchaka in :gh:`153906`.) + + re -- @@ -765,6 +798,15 @@ Build changes (Contributed by Stefano Rivera in :gh:`131372`.) +* Add the :option:`--with-curses` :program:`configure` option to select the + curses backend for the :mod:`curses` and :mod:`curses.panel` modules. + In addition to ``ncursesw`` and ``ncurses``, it can now build against the + system's native ``curses`` library (for example on NetBSD or Solaris), with + wide-character support when the library provides it. :option:`--without-curses` + excludes the modules from the build. + + (Contributed by Serhiy Storchaka in :gh:`136687`.) + C API changes ============= @@ -777,7 +819,8 @@ New features Porting to Python 3.16 ---------------------- -* TODO +* :c:func:`PyType_ClearCache` is now a no-op as the type cache is now + implemented per-type. It still returns the current version tag. Deprecated C APIs ----------------- diff --git a/Grammar/python.gram b/Grammar/python.gram index ac1b4a64f38c75a..2713ba9466a0b1d 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -568,7 +568,6 @@ real_number[expr_ty]: imaginary_number[expr_ty]: | imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } - | '+' imag=NUMBER { _PyPegen_ensure_imaginary(p, imag) } capture_pattern[pattern_ty]: | target=pattern_capture_target { _PyAST_MatchAs(NULL, target->v.Name.id, EXTRA) } diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 326254c335b4895..4c5a677e5543ece 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -246,6 +246,8 @@ struct _typeobject { * This function must escape to any code that can result in * the GC being run, such as Py_DECREF. */ _Py_iteritemfunc _tp_iteritem; + + void *_tp_cache; }; #define _Py_ATTR_CACHE_UNUSED (30000) // (see tp_versions_used) diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index 69659c48a3bf882..738e342d1407686 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -96,7 +96,9 @@ typedef struct _object_stats { uint64_t type_cache_misses; uint64_t type_cache_dunder_hits; uint64_t type_cache_dunder_misses; - uint64_t type_cache_collisions; + uint64_t type_cache_too_big; + uint64_t type_cache_invalidations; + uint64_t type_cache_resizes; /* Temporary value used during GC */ uint64_t object_visits; } ObjectStats; diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 7d952a1e52561a6..6df1c01f151f68e 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1544,6 +1544,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_filters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load_lazy_submodule)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fix_up_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8a8bbc3b6d05bf7..873344fbdcb67b0 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -267,6 +267,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_filters) STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) + STRUCT_FOR_ID(_find_and_load_lazy_submodule) STRUCT_FOR_ID(_fix_up_module) STRUCT_FOR_ID(_flags_) STRUCT_FOR_ID(_get_sourcefile) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index a1078828afa572e..669e328c266d00f 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -39,8 +39,13 @@ extern PyObject * _PyImport_GetAbsName( // Symbol is exported for the JIT on Windows builds. PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( PyThreadState *tstate, PyObject *lazy_import); -extern PyObject * _PyImport_TryLoadLazySubmodule( - PyObject *mod_name, PyObject *attr_name); +typedef enum { + _Py_LAZY_SUBMODULE_ERROR = -1, + _Py_LAZY_SUBMODULE_NOT_FOUND = 0, + _Py_LAZY_SUBMODULE_LOADED = 1, +} _PyLazySubmoduleImportResult; +extern _PyLazySubmoduleImportResult _PyImport_TryLoadLazySubmodule( + PyObject *mod_name, PyObject *attr_name, PyObject **result); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index d3efac906aeb933..0623adce693d465 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -560,23 +560,6 @@ struct _types_runtime_state { }; -// Type attribute lookup cache: speed up attribute and method lookups, -// see _PyType_Lookup(). -struct type_cache_entry { - unsigned int version; // initialized from type->tp_version_tag -#ifdef Py_GIL_DISABLED - _PySeqLock sequence; -#endif - PyObject *name; // reference to exactly a str or None - PyObject *value; // borrowed reference or NULL -}; - -#define MCACHE_SIZE_EXP 12 - -struct type_cache { - struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; -}; - typedef struct { PyTypeObject *type; int isbuiltin; @@ -591,6 +574,10 @@ typedef struct { are also some diagnostic uses for the list of weakrefs, so we still keep it. */ PyObject *tp_weaklist; + /* Per-interpreter attribute lookup cache (struct type_cache *). + For static builtin types the cache must be per-interpreter + because tp_dict and the values it stores are per-interpreter. */ + void *_tp_cache; } managed_static_type_state; #define TYPE_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ @@ -601,8 +588,6 @@ struct types_state { where all those lower numbers are used for core static types. */ unsigned int next_version_tag; - struct type_cache type_cache; - /* Every static builtin type is initialized for each interpreter during its own initialization, including for the main interpreter during global runtime initialization. This is done by calling diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 53ed570f99f118a..e0c5cfc02bbc359 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -310,9 +310,6 @@ Known values: You should always use the *upcoming* tag. For example, if 3.12a6 came out a week ago, I should put "Python 3.12a7" next to my new magic number. -Whenever PYC_MAGIC_NUMBER is changed, the ranges in the magic_values array in -PC/launcher.c must also be updated. - */ #define PYC_MAGIC_NUMBER 3704 diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 307ecdbaaefcea2..41786cb267c2e96 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -285,8 +285,6 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { return ((type->tp_flags) & feature) != 0; } -extern void _PyType_InitCache(PyInterpreterState *interp); - extern PyStatus _PyObject_InitState(PyInterpreterState *interp); extern void _PyObject_FiniState(PyInterpreterState *interp); extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 366d2d300fb4780..378f27ca17b5079 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1542,6 +1542,7 @@ extern "C" { INIT_ID(_filters), \ INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ + INIT_ID(_find_and_load_lazy_submodule), \ INIT_ID(_fix_up_module), \ INIT_ID(_flags_), \ INIT_ID(_get_sourcefile), \ diff --git a/Include/internal/pycore_typecache.h b/Include/internal/pycore_typecache.h new file mode 100644 index 000000000000000..76f87e8548d9c90 --- /dev/null +++ b/Include/internal/pycore_typecache.h @@ -0,0 +1,47 @@ +#ifndef PY_INTERNAL_TYPECACHE_H +#define PY_INTERNAL_TYPECACHE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_stackref.h" + + +#define _Py_TYPECACHE_MINSIZE (1 << 3) +#define _Py_TYPECACHE_MAXSIZE (1 << 16) + +struct type_cache_entry { + PyObject *name; // name of the attribute or method, interned string, NULL if the entry is empty + PyObject *value; // borrowed reference or NULL +}; + +// Per-type attribute lookup cache: speed up attribute and method lookups, +// see _PyTypeCache_Lookup(). +struct type_cache { + uint32_t mask; // mask for indexing into hashtable, i.e. size of hashtable is mask + 1 + uint32_t available; // number of available entries in hashtable + uint32_t used; // number of used entries in hashtable + unsigned int version_tag; // initialized from type->tp_version_tag + struct type_cache_entry hashtable[_Py_TYPECACHE_MINSIZE]; // hashtable entries +}; + +struct _PyTypeCacheLookupResult { + _PyStackRef value; // value is a stack reference to the cached attribute or method, or NULL if not found + int cache_hit; // 1 if the cache entry is valid and matches the type's version tag, 0 otherwise + unsigned int version_tag; // version tag of the type when the value was cached +}; + + +extern void _PyTypeCache_InitType(PyTypeObject *type); +extern void _PyTypeCache_Insert(PyTypeObject *type, PyObject *name, PyObject *value); +PyAPI_FUNC(struct _PyTypeCacheLookupResult) _PyTypeCache_Lookup(PyTypeObject *type, PyObject *name); +PyAPI_FUNC(void) _PyTypeCache_Invalidate(PyTypeObject *type); + +#ifdef __cplusplus +} +#endif +#endif /* PY_INTERNAL_TYPECACHE_H */ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index c8b888f1c395483..ecb3c78a35b9165 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -41,7 +41,6 @@ extern PyStatus _PyTypes_InitTypes(PyInterpreterState *); extern void _PyTypes_FiniTypes(PyInterpreterState *); extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp); extern void _PyTypes_Fini(PyInterpreterState *); -extern void _PyTypes_AfterFork(void); extern void _PyTypes_FiniCachedDescriptors(PyInterpreterState *); static inline PyObject ** diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 00d6297432b5fca..daf6840aa47f328 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -848,6 +848,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_find_and_load_lazy_submodule); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_fix_up_module); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/py_curses.h b/Include/py_curses.h index a5f5f3d76075827..85a540086c83844 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -43,6 +43,15 @@ # define PDC_NCMOUSE #endif +/* On Solaris/illumos, the SVr4 does "typedef char bool;", which + clashes with C's bool from . Define _BOOL to suppress it, and + include for the bool the header then needs. ncurses ignores + _BOOL. */ +#if defined(__sun) && !defined(_BOOL) +# include +# define _BOOL +#endif + #if defined(HAVE_NCURSESW_NCURSES_H) # include #elif defined(HAVE_NCURSESW_CURSES_H) diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index 7fc41a807dd5667..fa89996aa654291 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -307,9 +307,7 @@ With a new bytecode you must also change what is called the "magic number" for .pyc files: bump the value of the variable `MAGIC_NUMBER` in [`Lib/importlib/_bootstrap_external.py`](../Lib/importlib/_bootstrap_external.py). Changing this number will lead to all .pyc files with the old `MAGIC_NUMBER` -to be recompiled by the interpreter on import. Whenever `MAGIC_NUMBER` is -changed, the ranges in the `magic_values` array in -[`PC/launcher.c`](../PC/launcher.c) may also need to be updated. Changes to +to be recompiled by the interpreter on import. Changes to [`Lib/importlib/_bootstrap_external.py`](../Lib/importlib/_bootstrap_external.py) will take effect only after running `make regen-importlib`. diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 27eb7f13baca971..5f44a3aa05eb8f6 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -405,6 +405,7 @@ class Syntax(ThemeSection): keyword: str = ANSIColors.BOLD_BLUE keyword_constant: str = ANSIColors.BOLD_BLUE builtin: str = ANSIColors.CYAN + command: str = ANSIColors.BOLD_CYAN comment: str = ANSIColors.RED string: str = ANSIColors.GREEN number: str = ANSIColors.YELLOW diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index c169d0191bd8333..da557b254e52c48 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -68,6 +68,7 @@ def _clear_screen(): reader.scheduled_commands.append("clear_screen") +# Keep this in sync with _pyrepl.utils.COMMANDS REPL_COMMANDS = { "exit": _sitebuiltins.Quitter('exit', ''), "quit": _sitebuiltins.Quitter('quit' ,''), diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 230dae35af665ab..8d68f4f66f2980e 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -24,6 +24,8 @@ IDENTIFIERS_AFTER = frozenset({"def", "class"}) KEYWORD_CONSTANTS = frozenset({"True", "False", "None"}) BUILTINS = frozenset({str(name) for name in dir(builtins) if not name.startswith('_')}) +# Keep this in sync with _pyrepl.simple_interact.REPL_COMMANDS +COMMANDS = frozenset({"exit", "quit", "copyright", "help", "clear"}) def THEME(**kwargs): @@ -235,6 +237,13 @@ def gen_colors_from_token_stream( ): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "soft_keyword") + elif ( + token.string in COMMANDS + and (not prev_token or prev_token.type == T.INDENT) + and (not next_token or next_token.type == T.NEWLINE) + ): + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "command") elif ( token.string in BUILTINS and not (prev_token and prev_token.exact_type == T.DOT) diff --git a/Lib/argparse.py b/Lib/argparse.py index 29e6ebb9634261a..f8739d031e01c58 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -87,12 +87,17 @@ import os as _os -import re as _re import sys as _sys -from gettext import gettext as _ -from gettext import ngettext lazy import _colorize +lazy import copy +lazy import difflib +lazy import re as _re +lazy import shutil +lazy import textwrap +lazy import warnings +lazy from gettext import gettext as _ +lazy from gettext import ngettext SUPPRESS = '==SUPPRESS==' @@ -143,10 +148,8 @@ def _copy_items(items): return [] # The copy module is used only in the 'append' and 'append_const' # actions, and it is needed only when the default value isn't a list. - # Delay its import for speeding up the common case. if type(items) is list: return items[:] - import copy return copy.copy(items) @@ -186,7 +189,6 @@ def __init__( ): # default setting for width if width is None: - import shutil width = shutil.get_terminal_size().columns width -= 2 @@ -773,14 +775,38 @@ def _iter_indented_subactions(self, action): def _split_lines(self, text, width): text = self._whitespace_matcher.sub(' ', text).strip() - # The textwrap module is used only for formatting help. - # Delay its import for speeding up the common usage of argparse. - import textwrap - return textwrap.wrap(text, width) + decolored = self._decolor(text) + if decolored == text: + return textwrap.wrap(text, width) + + # gh-142035: colors inflate textwrap's length counts, so wrap + # the decolored text and re-apply colors per word; if textwrap + # split a word, keep the plain lines (colors can't be mapped). + plain = self._whitespace_matcher.sub(' ', decolored).strip() + if not plain: + # nothing visible to wrap (e.g. an empty interpolated value) + return [text] + plain_lines = textwrap.wrap(plain, width) + plain_words = plain.split() + colored_words = text.split() + # Drop escape-only tokens (e.g. an empty interpolated value). + if len(colored_words) != len(plain_words): + colored_words = [ + word for word in colored_words if self._decolor(word) + ] + colored_lines = [] + start = 0 + for plain_line in plain_lines: + plain_line_words = plain_line.split() + end = start + len(plain_line_words) + if plain_words[start:end] != plain_line_words: + return plain_lines + colored_lines.append(' '.join(colored_words[start:end])) + start = end + return colored_lines def _fill_text(self, text, width, indent): text = self._whitespace_matcher.sub(' ', text).strip() - import textwrap return textwrap.fill(text, width, initial_indent=indent, subsequent_indent=indent) @@ -1458,7 +1484,6 @@ class FileType(object): """ def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None): - import warnings warnings.warn( "FileType is deprecated. Simply open files after parsing arguments.", category=PendingDeprecationWarning, @@ -1865,7 +1890,6 @@ class _ArgumentGroup(_ActionsContainer): def __init__(self, container, title=None, description=None, **kwargs): if 'prefix_chars' in kwargs: - import warnings depr_msg = ( "The use of the undocumented 'prefix_chars' parameter in " "ArgumentParser.add_argument_group() is deprecated." @@ -2796,7 +2820,6 @@ def _check_value(self, action, value): if self.suggest_on_error and isinstance(value, str): if all(isinstance(choice, str) for choice in action.choices): - import difflib suggestions = difflib.get_close_matches(value, action.choices, 1) if suggestions: args['closest'] = suggestions[0] @@ -2936,8 +2959,6 @@ def _warning(self, message): def __getattr__(name): if name == "__version__": - from warnings import _deprecated - - _deprecated("__version__", remove=(3, 20)) + warnings._deprecated("__version__", remove=(3, 20)) return "1.1" # Do not change raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 7f0565d0b8ddc76..cbb052630d71ce9 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -99,7 +99,7 @@ def run(self): console.write(banner) - if not sys.flags.isolated and (startup_path := os.getenv("PYTHONSTARTUP")): + if not sys.flags.ignore_environment and (startup_path := os.getenv("PYTHONSTARTUP")): sys.audit("cpython.run_startup", startup_path) try: import tokenize @@ -200,7 +200,7 @@ def interrupt(self) -> None: sys.audit("cpython.run_stdin") - if os.getenv('PYTHON_BASIC_REPL'): + if not sys.flags.ignore_environment and os.getenv('PYTHON_BASIC_REPL'): CAN_USE_PYREPL = False else: try: @@ -212,11 +212,14 @@ def interrupt(self) -> None: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - repl_locals = {'asyncio': asyncio} - for key in {'__name__', '__package__', - '__loader__', '__spec__', - '__builtins__', '__file__'}: - repl_locals[key] = locals()[key] + repl_locals = { + 'asyncio': asyncio, + '__name__': __name__, + '__package__': None, + '__loader__': __loader__, + '__spec__': None, + '__builtins__': __builtins__, + } console = AsyncIOInteractiveConsole(repl_locals, loop) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 3836aaad326f62e..f26fba175b63cd7 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -590,7 +590,7 @@ async def shutdown_asyncgens(self): return_exceptions=True) for result, agen in zip(results, closing_agens): - if isinstance(result, Exception): + if isinstance(result, BaseException): self.call_exception_handler({ 'message': f'an error occurred during closing of ' f'asynchronous generator {agen!r}', diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 224b1883808a412..0423bc100b2d932 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -22,11 +22,10 @@ def __init__(self, loop, protocol, args, shell, self._proc = None self._pid = None self._returncode = None - self._exit_waiters = [] + self._exit_waiters = set() self._pending_calls = collections.deque() self._pipes = {} self._finished = False - self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -166,6 +165,9 @@ def kill(self): else: def send_signal(self, signal): self._check_proc() + if self._returncode is not None: + # The process already exited + return try: os.kill(self._proc.pid, signal) except ProcessLookupError: @@ -209,12 +211,19 @@ async def _connect_pipes(self, waiter): except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: + # Close any pipes that were already connected before the + # error/cancellation to avoid leaking file descriptors. + for proto in self._pipes.values(): + if proto is not None: + proto.pipe.close() + for raw_pipe in (proc.stdin, proc.stdout, proc.stderr): + if raw_pipe is not None: + raw_pipe.close() if waiter is not None and not waiter.cancelled(): waiter.set_exception(exc) else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) - self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -235,6 +244,7 @@ def _process_exited(self, returncode): if self._loop.get_debug(): logger.info('%r exited with return code %r', self, returncode) self._returncode = returncode + if self._proc.returncode is None: # asyncio uses a child watcher: copy the status into the Popen # object. On Python 3.6, it is required to avoid a ResourceWarning. @@ -243,6 +253,13 @@ def _process_exited(self, returncode): self._try_finish() + # gh-119710: Wake up futures waiting for wait() as soon as the process + # exits. + for waiter in self._exit_waiters: + if not waiter.done(): + waiter.set_result(returncode) + self._exit_waiters = None + async def _wait(self): """Wait until the process exit and return the process return code. @@ -251,22 +268,18 @@ async def _wait(self): return self._returncode waiter = self._loop.create_future() - self._exit_waiters.append(waiter) - return await waiter + self._exit_waiters.add(waiter) + try: + return await waiter + finally: + if self._exit_waiters is not None: + self._exit_waiters.discard(waiter) def _try_finish(self): assert not self._finished if self._returncode is None: return - if not self._pipes_connected: - # self._pipes_connected can be False if not all pipes were connected - # because either the process failed to start or the self._connect_pipes task - # got cancelled. In this broken state we consider all pipes disconnected and - # to avoid hanging forever in self._wait as otherwise _exit_waiters - # would never be woken up, we wake them up here. - for waiter in self._exit_waiters: - if not waiter.done(): - waiter.set_result(self._returncode) + if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True @@ -276,11 +289,6 @@ def _call_connection_lost(self, exc): try: self._protocol.connection_lost(exc) finally: - # wake up futures waiting for wait() - for waiter in self._exit_waiters: - if not waiter.done(): - waiter.set_result(self._returncode) - self._exit_waiters = None self._loop = None self._proc = None self._protocol = None diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index 35f7fa62140bd49..7e85e08c4291a06 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -62,7 +62,7 @@ def _build_graph_for_future( coro = coro.cr_await elif hasattr(coro, 'ag_await'): # A native async generator or duck-type compatible iterator - st.append(FrameCallGraphEntry(coro.cr_frame)) + st.append(FrameCallGraphEntry(coro.ag_frame)) coro = coro.ag_await else: break diff --git a/Lib/asyncio/protocols.py b/Lib/asyncio/protocols.py index 09987b164c66e57..c67a26c9e2986cc 100644 --- a/Lib/asyncio/protocols.py +++ b/Lib/asyncio/protocols.py @@ -199,6 +199,7 @@ def process_exited(self): def _feed_data_to_buffered_proto(proto, data): data_len = len(data) + start = 0 while data_len: buf = proto.get_buffer(data_len) buf_len = len(buf) @@ -206,11 +207,11 @@ def _feed_data_to_buffered_proto(proto, data): raise RuntimeError('get_buffer() returned an empty buffer') if buf_len >= data_len: - buf[:data_len] = data + buf[:data_len] = data[start:] if start else data proto.buffer_updated(data_len) return else: - buf[:buf_len] = data[:buf_len] + buf[:buf_len] = data[start:start + buf_len] proto.buffer_updated(buf_len) - data = data[buf_len:] - data_len = len(data) + start += buf_len + data_len -= buf_len diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 845aed4c6a3b352..b9e7b9eb29594fb 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -90,11 +90,7 @@ def task_done(task): return unhandled_exceptions.append(exc) - async def run_one_coro(ok_to_start, previous_failed) -> None: - # in eager tasks this waits for the calling task to append this task - # to running_tasks, in regular tasks this wait is a no-op that does - # not yield a future. See gh-124309. - await ok_to_start.wait() + async def run_one_coro(previous_failed) -> None: # Wait for the previous task to finish, or for delay seconds if previous_failed is not None: with contextlib.suppress(exceptions_mod.TimeoutError): @@ -110,14 +106,13 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: return # Start task that will run the next coroutine this_failed = locks.Event() - next_ok_to_start = locks.Event() - next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed)) + next_task = loop.create_task( + run_one_coro(this_failed), + eager_start=False, + ) futures.future_add_to_awaited_by(next_task, parent_task) running_tasks.add(next_task) next_task.add_done_callback(task_done) - # next_task has been appended to running_tasks so next_task is ok to - # start. - next_ok_to_start.set() # Prepare place to put this coroutine's exceptions if not won exceptions.append(None) assert len(exceptions) == this_index + 1 @@ -149,13 +144,11 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: propagate_cancellation_error = None try: - ok_to_start = locks.Event() - first_task = loop.create_task(run_one_coro(ok_to_start, None)) + first_task = loop.create_task(run_one_coro(None), eager_start=False) futures.future_add_to_awaited_by(first_task, parent_task) running_tasks.add(first_task) first_task.add_done_callback(task_done) - # first_task has been appended to running_tasks so first_task is ok to start. - ok_to_start.set() + # first_task has been appended to running_tasks before the event loop starts running it. propagate_cancellation_error = None # Make sure no tasks are left running if we leave this function while running_tasks: diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index 043359bbd03f8aa..26d7b755105c9d5 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -124,6 +124,11 @@ def __init__(self, transport, protocol, loop): self.stdout = protocol.stdout self.stderr = protocol.stderr self.pid = transport.get_pid() + self._communication_started = False + self._input = None + self._input_written = False + self._stdout_buf = bytearray() + self._stderr_buf = bytearray() def __repr__(self): return f'<{self.__class__.__name__} {self.pid}>' @@ -148,8 +153,9 @@ def kill(self): async def _feed_stdin(self, input): debug = self._loop.get_debug() try: - if input is not None: + if input is not None and not self._input_written: self.stdin.write(input) + self._input_written = True if debug: logger.debug( '%r communicate: feed stdin (%s bytes)', self, len(input)) @@ -172,22 +178,33 @@ async def _read_stream(self, fd): transport = self._transport.get_pipe_transport(fd) if fd == 2: stream = self.stderr + buf = self._stderr_buf else: assert fd == 1 stream = self.stdout + buf = self._stdout_buf if self._loop.get_debug(): name = 'stdout' if fd == 1 else 'stderr' logger.debug('%r communicate: read %s', self, name) - output = await stream.read() + # Append each block to the persistent buffer as soon as it is + # read so that no output is lost if this coroutine is cancelled. + while block := await stream.read(stream._limit): + buf += block if self._loop.get_debug(): name = 'stdout' if fd == 1 else 'stderr' logger.debug('%r communicate: close %s', self, name) transport.close() - return output async def communicate(self, input=None): + if self._communication_started: + if input: + raise ValueError( + "Cannot send input after starting communication") + else: + self._input = input + self._communication_started = True if self.stdin is not None: - stdin = self._feed_stdin(input) + stdin = self._feed_stdin(self._input) else: stdin = self._noop() if self.stdout is not None: @@ -198,8 +215,16 @@ async def communicate(self, input=None): stderr = self._read_stream(2) else: stderr = self._noop() - stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + await tasks.gather(stdin, stdout, stderr) await self.wait() + if self.stdout is not None: + stdout = self._stdout_buf.take_bytes() + else: + stdout = None + if self.stderr is not None: + stderr = self._stderr_buf.take_bytes() + else: + stderr = None return (stdout, stderr) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 93dd8994e1bcd51..bb98f0014f81760 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -213,13 +213,13 @@ async def _make_subprocess_transport(self, protocol, args, shell, raise except BaseException: transp.close() - await transp._wait() + await tasks.shield(transp._wait()) raise return transp def _child_watcher_callback(self, pid, returncode, transp): - self.call_soon_threadsafe(transp._process_exited, returncode) + transp._process_exited(returncode) async def create_unix_connection( self, protocol_factory, path=None, *, @@ -640,7 +640,8 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None): self._conn_lost = 0 self._closing = False # Set when close() or write_eof() called. - mode = os.fstat(self._fileno).st_mode + pipe_stat = os.fstat(self._fileno) + mode = pipe_stat.st_mode is_char = stat.S_ISCHR(mode) is_fifo = stat.S_ISFIFO(mode) is_socket = stat.S_ISSOCK(mode) @@ -657,7 +658,19 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None): # On AIX, the reader trick (to be notified when the read end of the # socket is closed) only works for sockets. On other platforms it # works for pipes and sockets. (Exception: OS X 10.4? Issue #19294.) - if is_socket or (is_fifo and not sys.platform.startswith("aix")): + # On macOS, the trick misfires for named FIFOs (but not for pipes + # created with os.pipe(), which have st_nlink == 0): the write end + # polls as readable whenever unread data sits in the FIFO, and no + # event is delivered when the read end is closed, so it can only + # ever report a false disconnection (gh-145030). The same xnu + # behaviour applies on iOS/tvOS/watchOS (sys.platform is not + # "darwin" there). + is_named_fifo_on_apple = ( + sys.platform in {"darwin", "ios", "tvos", "watchos"} + and is_fifo and pipe_stat.st_nlink > 0) + if is_socket or (is_fifo + and not sys.platform.startswith("aix") + and not is_named_fifo_on_apple): # only start reading when connection_made() has been called self._loop.call_soon(self._loop._add_reader, self._fileno, self._read_ready) @@ -930,6 +943,49 @@ def add_child_handler(self, pid, callback, *args): def _do_waitpid(self, loop, expected_pid, callback, args): assert expected_pid > 0 + if hasattr(os, 'waitid'): + # Wait for the child process using waitid() on platforms which support it. + # WNOWAIT is used to avoid reaping the child process, allowing the event loop to + # reap the child process with waitpid() later in event loop thread. + # This makes the reaping of the child and notification of the return code + # atomic with respect to the event loop thread. + try: + os.waitid(os.P_PID, expected_pid, os.WEXITED | os.WNOWAIT) + except ChildProcessError: + # The child process is already reaped + pass + if loop.is_closed(): + # loop is already closed, reap the zombie here so that it is not leaked. + pid, _ = self._reap(loop, expected_pid) + logger.warning("Loop %r that handles pid %r is closed", + loop, pid) + else: + try: + loop.call_soon_threadsafe( + self._reap_and_notify, loop, expected_pid, + callback, args) + except RuntimeError: + # The event loop was closed concurrently. + pid, _ = self._reap(loop, expected_pid) + logger.warning("Loop %r that handles pid %r is closed", + loop, pid) + else: + # Fallback for platforms that don't support waitid(): we have to + # reap the child here, which is racy with respect to send_signal() + pid, returncode = self._reap(loop, expected_pid) + if loop.is_closed(): + logger.warning("Loop %r that handles pid %r is closed", + loop, pid) + else: + loop.call_soon_threadsafe(callback, pid, returncode, *args) + + self._threads.pop(expected_pid) + + def _reap_and_notify(self, loop, expected_pid, callback, args): + pid, returncode = self._reap(loop, expected_pid) + callback(pid, returncode, *args) + + def _reap(self, loop, expected_pid): try: pid, status = os.waitpid(expected_pid, 0) except ChildProcessError: @@ -945,13 +1001,7 @@ def _do_waitpid(self, loop, expected_pid, callback, args): if loop.get_debug(): logger.debug('process %s exited with returncode %s', expected_pid, returncode) - - if loop.is_closed(): - logger.warning("Loop %r that handles pid %r is closed", loop, pid) - else: - loop.call_soon_threadsafe(callback, pid, returncode, *args) - - self._threads.pop(expected_pid) + return pid, returncode def can_use_pidfd(): if not hasattr(os, 'pidfd_open'): diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 8241c10b32d105c..efc7c0c158d3905 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -406,7 +406,7 @@ async def _make_subprocess_transport(self, protocol, args, shell, raise except BaseException: transp.close() - await transp._wait() + await tasks.shield(transp._wait()) raise return transp diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 20f1e728733fec6..5dbcac19e7a9272 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -553,6 +553,9 @@ class Counter(dict): or multiset. Elements are stored as dictionary keys and their counts are stored as dictionary values. + When constructed from a Mapping or Counter, the original object's + values will be used as the initial counts. + >>> c = Counter('abcdeabcdabcaba') # count elements from a string >>> c.most_common(3) # three most common elements diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py index c4477244e8327f9..26d7c9006375fb6 100644 --- a/Lib/compression/zstd/_zstdfile.py +++ b/Lib/compression/zstd/_zstdfile.py @@ -34,7 +34,7 @@ def __init__(self, file, /, mode='r', *, level=None, options=None, zstd_dict=None): """Open a Zstandard compressed file in binary mode. - *file* can be either an file-like object, or a file name to open. + *file* can be either a file-like object, or a file name to open. *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' for creating exclusively, or 'a' for appending. These can diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index 4e71331f9a0a594..43774066c5a9f00 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -703,5 +703,5 @@ def __exit__(self, exc_type, exc_val, exc_tb): class BrokenExecutor(RuntimeError): """ - Raised when a executor has become non-functional after a severe failure. + Raised when an executor has become non-functional after a severe failure. """ diff --git a/Lib/csv.py b/Lib/csv.py index b2aaf5fd9fa91e6..c66717dc1ee59e7 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -233,6 +233,11 @@ class Sniffer: "Sniffs" the format of a CSV file (i.e. delimiter, quotechar) Returns a Dialect object. ''' + # Characters which can be guessed as a delimiter if the delimiters + # argument is not specified. + _delimiter_candidates = [c for c in map(chr, range(128)) + if not c.isalnum()] + def __init__(self): # in case there is more than one possible delimiter self.preferred = [',', '\t', ';', ' ', ':'] @@ -240,214 +245,358 @@ def __init__(self): def sniff(self, sample, delimiters=None): """ - Returns a dialect (or None) corresponding to the sample + Analyze the sample and return a Dialect subclass reflecting the + parameters found. If the optional delimiters parameter is + given, it is interpreted as a string containing possible valid + delimiter characters. Raises Error if the dialect cannot be + determined. + + The dialect is guessed by parsing the sample with every + plausible combination of delimiter, quotechar and escapechar, + and choosing the combination which splits the sample into rows + with the most consistent number of fields. + + A large sample is parsed incrementally: first only its + beginning, then, after eliminating the combinations which are + clearly worse than the leader, a several times larger part, + and so on. + + If several combinations fit the sample equally well, the + delimiters listed in the preferred attribute are preferred, in + that order, no matter how many times each of them occurs. """ + import re + from collections import defaultdict - quotechar, doublequote, delimiter, skipinitialspace = \ - self._guess_quote_and_delimiter(sample, delimiters) - if not delimiter: - delimiter, skipinitialspace = self._guess_delimiter(sample, - delimiters) + if self._parses_as_single_column(sample): + # There is no delimiter to find; any combination could + # only find a bogus one inside the quoted fields. + raise Error("Could not determine delimiter") - if not delimiter: + chars = set(sample) + if delimiters is None: + delimiters = self._delimiter_candidates + delimiters = [d for d in delimiters + if d in chars and d not in '\r\n"\'\\'] + # Combinations to try, numbered by preference for breaking + # ties. The unquoted combinations are parsed from the start; + # the rest stay dormant until the quote character occurs at + # the start of a field (see _split_dormant). + groups = defaultdict(list) + order = 0 + # Only '\\' is tried as an escape character: others are not + # seen in the wild. + for escapechar in ('', '\\') if '\\' in chars else ('',): + for quotechar in '"', "'", '': + if quotechar and quotechar not in chars: + continue + for delimiter in delimiters: + groups[quotechar].append( + (order, delimiter, quotechar, escapechar)) + order += 1 + active = groups.pop('', []) + # Only non-empty groups were created; a plain dict cannot + # grow one by accident. + dormant = dict(groups) + + # The initial window should cover the minimal number of rows + # required for elimination (see _eliminate_worse) at a typical + # line length, so that the first round can already eliminate. + window = 2000 + # A line with its line break: '\r', '\n' or '\r\n' (the + # reader treats other line boundary characters as ordinary + # data, but does not support a bare '\r' inside a chunk). + # The \z alternative produces one final empty match. + line_re = re.compile(r'[^\r\n]*(?:\r\n|[\r\n]|\z)') + parsed = [] + lines = [] + first_round = True + while active or dormant: + end = min(window, len(sample)) + part = sample[:end] + lines = line_re.findall(part) + del lines[-1] + cut = not part.endswith(('\r', '\n')) + for quotechar in list(dormant): + activated, still = self._split_dormant( + part, quotechar, dormant[quotechar]) + active += activated + if still: + dormant[quotechar] = still + else: + del dormant[quotechar] + parsed = [(combo, self._try_dialect(lines, cut, *combo[1:])) + for combo in active] + if end == len(sample): + break + active = self._eliminate_worse(parsed, not first_round) + first_round = False + if len(active) <= 3: + # Quoted data most often leaves three survivors: the + # true dialect, its equally consistent unquoted shadow, + # and one accident. Parsing the whole sample with them + # is cheaper than another elimination round. + window = len(sample) + else: + # Too small a factor would increase the total + # re-parsing cost, too large -- the cost of the next + # round if this one did not eliminate enough. + window *= 4 + + best = None + best_score = None + for combo, rows in sorted(parsed): + if rows is None: + continue + _, delimiter, quotechar, escapechar = combo + nfields, share = self._modal_share(rows) + if nfields < 2: + # The delimiter does not delimit anything. + continue + try: + preference = -self.preferred.index(delimiter) + except ValueError: + preference = -len(self.preferred) + # A successful quoted parse is direct evidence; the preferred + # delimiters list is only a nudge. + score = (share, len(rows), bool(quotechar), preference) + if best_score is None or score > best_score: + best_score = score + best = combo[1:] + + if best is None: raise Error("Could not determine delimiter") + delimiter, quotechar, escapechar = best + doublequote = self._detect_doublequote(lines, *best) + skipinitialspace = self._detect_skipinitialspace(lines, *best, + doublequote) class dialect(Dialect): _name = "sniffed" lineterminator = '\r\n' quoting = QUOTE_MINIMAL - # escapechar = '' - dialect.doublequote = doublequote dialect.delimiter = delimiter # _csv.reader won't accept a quotechar of '' dialect.quotechar = quotechar or '"' + dialect.escapechar = escapechar or None + dialect.doublequote = doublequote dialect.skipinitialspace = skipinitialspace return dialect - - def _guess_quote_and_delimiter(self, data, delimiters): + def _parses_as_single_column(self, sample): """ - Looks for text enclosed between two identical quotes - (the probable quotechar) which are preceded and followed - by the same character (the probable delimiter). - For example: - ,'some text', - The quote with the most wins, same with the delimiter. - If there is no quotechar the delimiter can't be determined - this way. + True if the whole sample parses as a single column of quoted + fields (the last one may be cut off in the middle), so there + is no delimiter to find. """ import re - matches = [] - for restr in (r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?P=delim)', # ,".*?", - r'(?:^|\n)(?P["\']).*?(?P=quote)(?P[^\w\n"\'])(?P ?)', # ".*?", - r'(?P[^\w\n"\'])(?P ?)(?P["\']).*?(?P=quote)(?:$|\n)', # ,".*?" - r'(?:^|\n)(?P["\']).*?(?P=quote)(?:$|\n)'): # ".*?" (no delim, no space) - regexp = re.compile(restr, re.DOTALL | re.MULTILINE) - matches = regexp.findall(data) - if matches: - break - - if not matches: - # (quotechar, doublequote, delimiter, skipinitialspace) - return ('', False, None, 0) - quotes = {} - delims = {} - spaces = 0 - groupindex = regexp.groupindex - for m in matches: - n = groupindex['quote'] - 1 - key = m[n] - if key: - quotes[key] = quotes.get(key, 0) + 1 - try: - n = groupindex['delim'] - 1 - key = m[n] - except KeyError: - continue - if key and (delimiters is None or key in delimiters): - delims[key] = delims.get(key, 0) + 1 - try: - n = groupindex['space'] - 1 - except KeyError: - continue - if m[n]: - spaces += 1 - - quotechar = max(quotes, key=quotes.get) - - if delims: - delim = max(delims, key=delims.get) - skipinitialspace = delims[delim] == spaces - if delim == '\n': # most likely a file with a single column - delim = '' - else: - # there is *no* delimiter, it's a single column of quoted data - delim = '' - skipinitialspace = 0 - - # if we see an extra quote between delimiters, we've got a - # double quoted format - dq_regexp = re.compile( - r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ - {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE) - + for q in '"', "'": + if q in sample: + row_re = (fr' *+{q}(?:[^{q}]|{q}{q})*+' + fr'(?:{q} *+(?:[\r\n]++|\z)|\z)') + if re.fullmatch(fr'(?:{row_re})++', sample): + return True + return False + def _split_dormant(self, part, quotechar, combos): + """ + Split the dormant combinations into those ready for trial + parsing and the rest. + + A combination is ready when its quote character occurs in + *part* at the start of a field, i.e. at the start of a line or + after its delimiter; until then parsing would not differ from + the unquoted variant. Spaces before the quote are allowed even + for the space delimiter: a false activation only costs a trial + parse. + """ + import re - if dq_regexp.search(data): - doublequote = True + remaining = {combo[1] for combo in combos} + found = set() + pos = 0 + while remaining: + # Include only the delimiters not found yet, so that the + # search skips over the found ones; the compiled patterns + # come from the re cache. + cls = re.escape(''.join(sorted(remaining))) + m = re.compile(fr'(?:^|([\r\n{cls}]))' + fr' *{quotechar}').search(part, pos) + if m is None: + break + pre = m[1] + if pre is None or pre in '\r\n': + # A quote at the start of a line starts a field for + # every delimiter. + found |= remaining + break + found.add(pre) + remaining.discard(pre) + pos = m.end() + activated = [combo for combo in combos if combo[1] in found] + still_dormant = [combo for combo in combos if combo[1] not in found] + return activated, still_dormant + + def _make_reader(self, lines, delimiter, quotechar, escapechar, + doublequote=True, skipinitialspace=None): + """ + Create a reader for trial parsing. quotechar '' means no + quoting and escapechar '' means no escape character. + """ + if skipinitialspace is None: + # Be lenient to spaces after a delimiter, unless the + # delimiter is a space itself. + skipinitialspace = delimiter != ' ' + return reader(lines, delimiter=delimiter, + quotechar=quotechar or '"', + quoting=QUOTE_MINIMAL if quotechar else QUOTE_NONE, + escapechar=escapechar or None, + doublequote=doublequote, + skipinitialspace=skipinitialspace, + strict=True) + + def _try_dialect(self, lines, cut, delimiter, quotechar, escapechar): + """ + Parse the sample, pre-split into *lines*, and return the list + of the number of fields in every parsed row, or None if not a + single row was parsed. + + If the sample cannot be parsed to the end (for example it is + cut off in the middle of a quoted field, or the combination + does not fit the sample), the rows parsed so far are counted. + The last row is not counted if *cut* is true: the sample can + be cut off in the middle of it. + """ + rows = [] + try: + rows.extend(map(len, self._make_reader(lines, delimiter, + quotechar, escapechar))) + except Error: + # The row which failed to parse is not counted. + pass else: - doublequote = False + if cut and len(rows) > 1: + rows.pop() + if 0 in rows: + # Blank lines produce empty rows. + rows = [nfields for nfields in rows if nfields] + return rows or None + + def _eliminate_worse(self, parsed, judge_hopeless): + """ + Return the combinations from *parsed* (a list of (combination, + rows) pairs) without those which are clearly worse than the + leader. Combinations with too few parsed rows (e.g. if the + parsed part ends in the middle of a large quoted field) are + not judged yet. + + If *judge_hopeless* is false, keep the combinations whose + delimiter does not delimit anything. Unlike the comparison + with the leader, which self-normalizes when the parsed part is + not representative, this verdict is absolute and irreversible, + so it is not trusted to the first part, which covers the least + representative beginning of the sample (titles, headers, + preamble). + """ + # Judging a combination by fewer rows is too noisy. + min_rows = 16 + hopeless = set() + scores = {} + for combo, rows in parsed: + if rows is not None and len(rows) >= min_rows: + nfields, share = self._modal_share(rows) + if nfields < 2: + if judge_hopeless: + hopeless.add(combo) + else: + scores[combo] = share + threshold = max(scores.values(), default=0.0) - 0.1 + return [combo for combo, _ in parsed + if combo not in hopeless + and scores.get(combo, threshold) >= threshold] - return (quotechar, doublequote, delim, skipinitialspace) + def _modal_share(self, rows): + """ + The most common number of fields in a row and its share of all + rows. Prefer the smaller number of fields in the case of a + tie: a candidate delimiter which delimits only half of the rows + is not convincing. + """ + from collections import Counter + counts = Counter(rows) + nfields = max(counts, key=lambda n: (counts[n], -n)) + return nfields, counts[nfields] / len(rows) - def _guess_delimiter(self, data, delimiters): + def _detect_doublequote(self, lines, delimiter, quotechar, escapechar): """ - The delimiter /should/ occur the same number of times on - each row. However, due to malformed data, it may not. We don't want - an all or nothing approach, so we allow for small variations in this - number. - 1) build a table of the frequency of each character on every line. - 2) build a table of frequencies of this frequency (meta-frequency?), - e.g. 'x occurred 5 times in 10 rows, 6 times in 1000 rows, - 7 times in 2 rows' - 3) use the mode of the meta-frequency to determine the /expected/ - frequency for that character - 4) find out how often the character actually meets that goal - 5) the character that best meets its goal is the delimiter - For performance reasons, the data is evaluated in chunks, so it can - try and evaluate the smallest portion of the data possible, evaluating - additional chunks as necessary. + True if a doubled quote character represents a single quote + character in the sample: interpreting it so changes the result + of parsing. """ - from collections import Counter, defaultdict - - data = list(filter(None, data.split('\n'))) - - # build frequency tables - chunkLength = min(10, len(data)) - iteration = 0 - num_lines = 0 - # {char -> {count_per_line -> num_lines_with_that_count}} - char_frequency = defaultdict(Counter) - modes = {} - delims = {} - start, end = 0, chunkLength - while start < len(data): - iteration += 1 - for line in data[start:end]: - num_lines += 1 - for char, count in Counter(line).items(): - if char.isascii(): - char_frequency[char][count] += 1 - - for char, counts in char_frequency.items(): - items = list(counts.items()) - missed_lines = num_lines - sum(counts.values()) - if missed_lines: - # Store the number of lines 'char' was missing from. - items.append((0, missed_lines)) - if len(items) == 1 and items[0][0] == 0: - continue - # get the mode of the frequencies - if len(items) > 1: - modes[char] = max(items, key=lambda x: x[1]) - # adjust the mode - subtract the sum of all - # other frequencies - items.remove(modes[char]) - modes[char] = (modes[char][0], modes[char][1] - - sum(item[1] for item in items)) - else: - modes[char] = items[0] - - # build a list of possible delimiters - modeList = modes.items() - total = float(min(chunkLength * iteration, len(data))) - # (rows of consistent data) / (number of rows) = 100% - consistency = 1.0 - # minimum consistency threshold - threshold = 0.9 - while len(delims) == 0 and consistency >= threshold: - for k, v in modeList: - if v[0] > 0 and v[1] > 0: - if ((v[1]/total) >= consistency and - (delimiters is None or k in delimiters)): - delims[k] = v - consistency -= 0.01 - - if len(delims) == 1: - delim = list(delims.keys())[0] - skipinitialspace = (data[0].count(delim) == - data[0].count("%c " % delim)) - return (delim, skipinitialspace) - - # analyze another chunkLength lines - start = end - end += chunkLength - - if not delims: - return ('', 0) - - # if there's more than one, fall back to a 'preferred' list - if len(delims) > 1: - for d in self.preferred: - if d in delims.keys(): - skipinitialspace = (data[0].count(d) == - data[0].count("%c " % d)) - return (d, skipinitialspace) - - # nothing else indicates a preference, pick the character that - # dominates(?) - items = [(v,k) for (k,v) in delims.items()] - items.sort() - delim = items[-1][1] - - skipinitialspace = (data[0].count(delim) == - data[0].count("%c " % delim)) - return (delim, skipinitialspace) - + if not quotechar or not any(quotechar * 2 in line + for line in lines): + return False + readers = [self._make_reader( + lines, delimiter, quotechar, escapechar, + doublequote=doublequote) + for doublequote in (False, True)] + while True: + rows = [] + for rdr in readers: + try: + rows.append(next(rdr)) + except (StopIteration, Error): + # Ending cleanly and failing are equivalent here: + # after equal rows both readers are at the same + # position, so they cannot end for different + # reasons. + rows.append(None) + if rows[0] != rows[1]: + return True + if rows == [None, None]: + return False + + def _detect_skipinitialspace(self, lines, delimiter, quotechar, + escapechar, doublequote): + """ + Detect whether the spaces following a delimiter are a part of + the format or of the data. + """ + results = [] + for skipinitialspace in False, True: + rows = [] + try: + rows.extend(self._make_reader( + lines, delimiter, quotechar, escapechar, + doublequote=doublequote, + skipinitialspace=skipinitialspace)) + except Error: + # Keep the rows parsed before the error. + pass + results.append([row for row in rows if row]) + if results[0] == results[1]: + return False # No evidence. + counts = [[len(row) for row in rows] for rows in results] + if counts[0] != counts[1]: + # Prefer the more consistent row widths. + return len(set(counts[1])) <= len(set(counts[0])) + # Only some spaces are stripped. A field differs only if + # a space was skipped at its start, which tells the padding + # apart from the spaces inside quoted or escaped fields. + if not all(kept_field != skipped_field + for kept_row, skipped_row in zip(*results) + for kept_field, skipped_field in zip(kept_row[1:], + skipped_row[1:])): + return False + # The first field of a row is commonly not padded ('a, b, c'), + # so the first fields need only agree with each other. + first = [kept_row[0] != skipped_row[0] + for kept_row, skipped_row in zip(*results)] + return all(first) or not any(first) def has_header(self, sample): # Creates a dictionary of types of data in each column. If any diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 51c0d441bc8df0e..c0a578c86549ec9 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -13,56 +13,19 @@ # find_library(name) returns the pathname of a library, or None. if os.name == "nt": - - def _get_build_version(): - """Return the version of MSVC that was used to build Python. - - For Python 2.3 and up, the version number is included in - sys.version. For earlier versions, assume the compiler is MSVC 6. - """ - # This function was copied from Lib/distutils/msvccompiler.py - prefix = "MSC v." - i = sys.version.find(prefix) - if i == -1: - return 6 - i = i + len(prefix) - s, rest = sys.version[i:].split(" ", 1) - majorVersion = int(s[:-2]) - 6 - if majorVersion >= 13: - majorVersion += 1 - minorVersion = int(s[2:3]) / 10.0 - # I don't think paths are affected by minor version in version 6 - if majorVersion == 6: - minorVersion = 0 - if majorVersion >= 6: - return majorVersion + minorVersion - # else we don't know what version of the compiler this is - return None - def find_msvcrt(): - """Return the name of the VC runtime dll""" - version = _get_build_version() - if version is None: - # better be safe than sorry - return None - if version <= 6: - clibname = 'msvcrt' - elif version <= 13: - clibname = 'msvcr%d' % (version * 10) - else: - # CRT is no longer directly loadable. See issue23606 for the - # discussion about alternative approaches. - return None - - # If python was built with in debug mode - import importlib.machinery - if '_d.pyd' in importlib.machinery.EXTENSION_SUFFIXES: - clibname += 'd' - return clibname+'.dll' + """Return the name of the VC runtime dll. + This is soft deprecated as of Python 3.16.""" + # See gh-154199. In short, this function wasn't able to return + # newer msvcrt versions (because there was no single msvcrt DLL), + # and the versions that are a single DLL are no longer supported + # by Microsoft. + return None def find_library(name): if name in ('c', 'm'): - return find_msvcrt() + # See gh-67794; there is no single VC runtime DLL anymore. + return None # See MSDN for the REAL search order. for directory in os.environ['PATH'].split(os.pathsep): fname = os.path.join(directory, name) @@ -429,8 +392,7 @@ def _findLib_ld(name): result = None try: p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) + stderr=subprocess.PIPE) out, _ = p.communicate() res = re.findall(expr, os.fsdecode(out)) for file in res: @@ -512,7 +474,8 @@ def _process_struct(decorated_class, /, *, align, layout, endian, pack): fields.extend(decorated_class._fields_) anonymous.extend(decorated_class._anonymous_) - for name, hint in annotationlib.get_annotations(decorated_class).items(): + annotations = annotationlib.get_annotations(decorated_class, eval_str=True) + for name, hint in annotations.items(): if get_origin(hint) is ClassVar: continue @@ -531,7 +494,8 @@ def _process_struct(decorated_class, /, *, align, layout, endian, pack): else: field.append(hint) - fields.append(field) + # _fields_ is a list of tuples + fields.append(tuple(field)) if endian == 'big': endian_class = BigEndianStructure @@ -574,6 +538,26 @@ def inner(decorated_class): return process_the_struct(class_or_none) + +def wrap_dll_function(dll): + def decorator(func): + name = func.__name__ + ptr = getattr(dll, name) + annotations = annotationlib.get_annotations(func, eval_str=True) + + try: + restype = annotations.pop("return") + except KeyError as error: + raise ValueError(f"{name!r} missing return type annotation") from error + + ptr.restype = restype + ptr.argtypes = tuple(annotations.values()) + functools.update_wrapper(ptr, func, updated=()) + + return ptr + + return decorator + ################################################################ # test code diff --git a/Lib/difflib.py b/Lib/difflib.py index ae8b284b4d36474..95ba8fd782c6c3c 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -2016,6 +2016,8 @@ def make_table(self,fromlines,tolines,fromdesc='',todesc='',context=False, # change tabs to spaces before it gets more difficult after we insert # markup + # it also removes trailing newlines, causing some diffs to be missed + # see: gh-71896 fromlines,tolines = self._tab_newline_replace(fromlines,tolines) # create diffs iterator which generates side by side from/to data diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index c68fd304ea4235e..696a3b2f8f1bc23 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -244,6 +244,7 @@ def test_font_set(self): def test_set_samples(self): d = self.page + self.addCleanup(setattr, d, 'set_samples', Func()) # Re-mask for other tests. del d.set_samples # Unmask method for test orig_samples = d.font_sample, d.highlight_sample d.font_sample, d.highlight_sample = {}, {} @@ -257,7 +258,6 @@ def test_set_samples(self): self.assertTrue(d.font_sample == d.highlight_sample == expected) d.font_sample, d.highlight_sample = orig_samples - d.set_samples = Func() # Re-mask for other tests. class HighPageTest(unittest.TestCase): @@ -307,7 +307,7 @@ def test_load_theme_cfg(self): # builtinlist sets variable builtin_name to the CurrentTheme default. eq(d.builtin_name.get(), 'IDLE Classic') eq(d.custom_name.get(), '- no custom themes -') - eq(d.custom_theme_on.state(), ('disabled',)) + self.assertIn('disabled', d.custom_theme_on.state()) eq(d.set_theme_type.called, 1) eq(d.paint_theme_sample.called, 1) eq(d.set_highlight_target.called, 1) @@ -495,6 +495,7 @@ def test_highlight_sample_b1_motion(self): def test_set_theme_type(self): eq = self.assertEqual d = self.page + self.addCleanup(setattr, d, 'set_theme_type', Func()) # Re-mask method. del d.set_theme_type # Builtin theme selected. @@ -502,16 +503,15 @@ def test_set_theme_type(self): d.set_theme_type() eq(d.builtinlist['state'], NORMAL) eq(d.customlist['state'], DISABLED) - eq(d.button_delete_custom.state(), ('disabled',)) + self.assertIn('disabled', d.button_delete_custom.state()) # Custom theme selected. d.theme_source.set(False) d.set_theme_type() eq(d.builtinlist['state'], DISABLED) - eq(d.custom_theme_on.state(), ('selected',)) + self.assertNotIn('disabled', d.custom_theme_on.state()) eq(d.customlist['state'], NORMAL) - eq(d.button_delete_custom.state(), ()) - d.set_theme_type = Func() + self.assertNotIn('disabled', d.button_delete_custom.state()) def test_get_color(self): eq = self.assertEqual @@ -637,24 +637,23 @@ def test_create_new_and_save_new(self): def test_set_highlight_target(self): eq = self.assertEqual d = self.page + self.addCleanup(setattr, d, 'set_highlight_target', Func()) # Re-mask method. del d.set_highlight_target # Target is cursor. d.highlight_target.set('Cursor') - eq(d.fg_on.state(), ('disabled', 'selected')) - eq(d.bg_on.state(), ('disabled',)) + self.assertIn('disabled', d.fg_on.state()) + self.assertIn('disabled', d.bg_on.state()) self.assertTrue(d.fg_bg_toggle) eq(d.set_color_sample.called, 1) # Target is not cursor. d.highlight_target.set('Comment') - eq(d.fg_on.state(), ('selected',)) - eq(d.bg_on.state(), ()) + self.assertNotIn('disabled', d.fg_on.state()) + self.assertNotIn('disabled', d.bg_on.state()) self.assertTrue(d.fg_bg_toggle) eq(d.set_color_sample.called, 2) - d.set_highlight_target = Func() - def test_set_color_sample_binding(self): d = self.page scs = d.set_color_sample @@ -667,6 +666,7 @@ def test_set_color_sample_binding(self): def test_set_color_sample(self): d = self.page + self.addCleanup(setattr, d, 'set_color_sample', Func()) # Re-mask method. del d.set_color_sample d.highlight_target.set('Selected Text') d.fg_bg_toggle.set(True) @@ -674,7 +674,6 @@ def test_set_color_sample(self): self.assertEqual( d.style.lookup(d.frame_color_set['style'], 'background'), d.highlight_sample.tag_cget('hilite', 'foreground')) - d.set_color_sample = Func() def test_paint_theme_sample(self): eq = self.assertEqual @@ -745,7 +744,7 @@ def test_delete_custom(self): eq(yesno.called, 2) self.assertNotIn(theme_name, highpage) eq(idleConf.GetSectionList('user', 'highlight'), [theme_name2]) - eq(d.custom_theme_on.state(), ()) + self.assertNotIn('disabled', d.custom_theme_on.state()) eq(d.custom_name.get(), theme_name2) eq(dialog.deactivate_current_config.called, 1) eq(dialog.activate_config_changes.called, 1) @@ -758,7 +757,7 @@ def test_delete_custom(self): eq(yesno.called, 3) self.assertNotIn(theme_name, highpage) eq(idleConf.GetSectionList('user', 'highlight'), []) - eq(d.custom_theme_on.state(), ('disabled',)) + self.assertIn('disabled', d.custom_theme_on.state()) eq(d.custom_name.get(), '- no custom themes -') eq(dialog.deactivate_current_config.called, 2) eq(dialog.activate_config_changes.called, 2) @@ -809,7 +808,7 @@ def test_load_key_cfg(self): # builtinlist sets variable builtin_name to the CurrentKeys default. eq(d.builtin_name.get(), 'IDLE Classic OSX') eq(d.custom_name.get(), '- no custom keys -') - eq(d.custom_keyset_on.state(), ('disabled',)) + self.assertIn('disabled', d.custom_keyset_on.state()) eq(d.set_keys_type.called, 1) eq(d.load_keys_list.called, 1) eq(d.load_keys_list.args, ('IDLE Classic OSX', )) @@ -930,6 +929,7 @@ def test_keybinding(self): def test_set_keys_type(self): eq = self.assertEqual d = self.page + self.addCleanup(setattr, d, 'set_keys_type', Func()) # Re-mask method. del d.set_keys_type # Builtin keyset selected. @@ -937,16 +937,15 @@ def test_set_keys_type(self): d.set_keys_type() eq(d.builtinlist['state'], NORMAL) eq(d.customlist['state'], DISABLED) - eq(d.button_delete_custom_keys.state(), ('disabled',)) + self.assertIn('disabled', d.button_delete_custom_keys.state()) # Custom keyset selected. d.keyset_source.set(False) d.set_keys_type() eq(d.builtinlist['state'], DISABLED) - eq(d.custom_keyset_on.state(), ('selected',)) + self.assertNotIn('disabled', d.custom_keyset_on.state()) eq(d.customlist['state'], NORMAL) - eq(d.button_delete_custom_keys.state(), ()) - d.set_keys_type = Func() + self.assertNotIn('disabled', d.button_delete_custom_keys.state()) def test_get_new_keys(self): eq = self.assertEqual @@ -1043,7 +1042,7 @@ def test_on_bindingslist_select(self): b.event_generate('', x=x, y=y) b.event_generate('', x=x, y=y) self.assertEqual(b.get('anchor'), 'find') - self.assertEqual(d.button_new_keys.state(), ()) + self.assertNotIn('disabled', d.button_new_keys.state()) def test_create_new_key_set_and_save_new_key_set(self): eq = self.assertEqual @@ -1080,6 +1079,7 @@ def test_load_keys_list(self): eq = self.assertEqual d = self.page gks = idleConf.GetKeySet = Func() + self.addCleanup(setattr, d, 'load_keys_list', Func()) # Re-mask method. del d.load_keys_list b = d.bindingslist @@ -1115,7 +1115,6 @@ def test_load_keys_list(self): eq(b.get(0, 'end'), expected) eq(b.get('anchor'), 'spam - ') eq(b.curselection(), (2, )) - d.load_keys_list = Func() del idleConf.GetKeySet @@ -1156,7 +1155,7 @@ def test_delete_custom_keys(self): eq(yesno.called, 2) self.assertNotIn(keyset_name, keyspage) eq(idleConf.GetSectionList('user', 'keys'), [keyset_name2]) - eq(d.custom_keyset_on.state(), ()) + self.assertNotIn('disabled', d.custom_keyset_on.state()) eq(d.custom_name.get(), keyset_name2) eq(dialog.deactivate_current_config.called, 1) eq(dialog.activate_config_changes.called, 1) @@ -1169,7 +1168,7 @@ def test_delete_custom_keys(self): eq(yesno.called, 3) self.assertNotIn(keyset_name, keyspage) eq(idleConf.GetSectionList('user', 'keys'), []) - eq(d.custom_keyset_on.state(), ('disabled',)) + self.assertIn('disabled', d.custom_keyset_on.state()) eq(d.custom_name.get(), '- no custom keys -') eq(dialog.deactivate_current_config.called, 2) eq(dialog.activate_config_changes.called, 2) @@ -1365,25 +1364,25 @@ def test_set_add_delete_state(self): # Call with 0 items, 1 unselected item, 1 selected item. eq = self.assertEqual fr = self.frame + self.addCleanup(setattr, fr, 'set_add_delete_state', Func()) # Re-mask method. del fr.set_add_delete_state # Unmask method. sad = fr.set_add_delete_state h = fr.helplist h.delete(0, 'end') sad() - eq(fr.button_helplist_edit.state(), ('disabled',)) - eq(fr.button_helplist_remove.state(), ('disabled',)) + self.assertIn('disabled', fr.button_helplist_edit.state()) + self.assertIn('disabled', fr.button_helplist_remove.state()) h.insert(0, 'source') sad() - eq(fr.button_helplist_edit.state(), ('disabled',)) - eq(fr.button_helplist_remove.state(), ('disabled',)) + self.assertIn('disabled', fr.button_helplist_edit.state()) + self.assertIn('disabled', fr.button_helplist_remove.state()) h.selection_set(0) sad() - eq(fr.button_helplist_edit.state(), ()) - eq(fr.button_helplist_remove.state(), ()) - fr.set_add_delete_state = Func() # Mask method. + self.assertNotIn('disabled', fr.button_helplist_edit.state()) + self.assertNotIn('disabled', fr.button_helplist_remove.state()) def test_helplist_item_add(self): # Call without and twice with HelpSource result. @@ -1461,6 +1460,7 @@ def test_helplist_item_remove(self): def test_update_help_changes(self): fr = self.frame + self.addCleanup(setattr, fr, 'update_help_changes', Func()) # Re-mask method. del fr.update_help_changes fr.user_helplist.clear() fr.user_helplist.append(('name1', 'file1')) @@ -1469,7 +1469,6 @@ def test_update_help_changes(self): fr.update_help_changes() self.assertEqual(mainpage['HelpFiles'], {'1': 'name1;file1', '2': 'name2;file2'}) - fr.update_help_changes = Func() class VarTraceTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index dc3431d8d8ef887..df84554c4f1a515 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -689,11 +689,15 @@ def test_mousewheel(self): last_lineno = get_end_linenumber(text) self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) - # Delta for , whose meaning is platform-dependent. + # Simulate a mouse wheel notch. Tk 8.7 replaced the X11 + # / wheel events with (whose delta is + # platform-dependent); older Tk on X11 still uses the button events. + x11_buttons = (sidebar.canvas._windowingsystem == 'x11' + and tk.TkVersion < 8.7) delta = 1 if sidebar.canvas._windowingsystem == 'aqua' else 120 # Scroll up. - if sidebar.canvas._windowingsystem == 'x11': + if x11_buttons: sidebar.canvas.event_generate('', x=0, y=0) else: sidebar.canvas.event_generate('', x=0, y=0, delta=delta) @@ -701,7 +705,7 @@ def test_mousewheel(self): self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) # Scroll back down. - if sidebar.canvas._windowingsystem == 'x11': + if x11_buttons: sidebar.canvas.event_generate('', x=0, y=0) else: sidebar.canvas.event_generate('', x=0, y=0, delta=-delta) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index eb1686a5c8217c6..081bb98ec2a9723 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1254,7 +1254,7 @@ def _sanity_check(name, package, level): _ERR_MSG_PREFIX = 'No module named ' -def _find_and_load_unlocked(name, import_): +def _find_and_load_unlocked(name, import_, *, lazy_submodule=False): path = None sys.audit( "import", @@ -1277,6 +1277,8 @@ def _find_and_load_unlocked(name, import_): try: path = parent_module.__path__ except AttributeError: + if lazy_submodule: + return None msg = f'{_ERR_MSG_PREFIX}{name!r}; {parent!r} is not a package' raise ModuleNotFoundError(msg, name=name) from None parent_spec = parent_module.__spec__ @@ -1289,6 +1291,8 @@ def _find_and_load_unlocked(name, import_): child = name.rpartition('.')[2] spec = _find_spec(name, path) if spec is None: + if lazy_submodule: + return None raise ModuleNotFoundError(f'{_ERR_MSG_PREFIX}{name!r}', name=name) else: if parent_spec: @@ -1320,7 +1324,7 @@ def _find_and_load_unlocked(name, import_): _NEEDS_LOADING = object() -def _find_and_load(name, import_): +def _find_and_load(name, import_, *, lazy_submodule=False): """Find and load the module.""" # Optimization: we avoid unneeded module locking if the module @@ -1337,7 +1341,8 @@ def _find_and_load(name, import_): with lock_manager: module = sys.modules.get(name, _NEEDS_LOADING) if module is _NEEDS_LOADING: - return _find_and_load_unlocked(name, import_) + return _find_and_load_unlocked( + name, import_, lazy_submodule=lazy_submodule) # Optimization: only call _bootstrap._lock_unlock_module() if # module.__spec__._initializing is True. @@ -1351,7 +1356,7 @@ def _find_and_load(name, import_): # to preserve normal semantics: the caller gets the exception from # the actual import failure rather than a synthetic error. if sys.modules.get(name) is not module: - return _find_and_load(name, import_) + return _find_and_load(name, import_, lazy_submodule=lazy_submodule) if module is None: message = f'import of {name} halted; None in sys.modules' @@ -1360,6 +1365,10 @@ def _find_and_load(name, import_): return module +def _find_and_load_lazy_submodule(name, import_): + return _find_and_load(name, import_, lazy_submodule=True) + + def _gcd_import(name, package=None, level=0): """Import and return the module based on its name, the package the call is being made from, and the level adjustment. diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 9ca127ad9c7d0fd..ec68379e76f6f31 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -1,6 +1,7 @@ """Abstract base classes related to import.""" from . import _bootstrap_external from . import machinery + try: import _frozen_importlib except ImportError as exc: @@ -11,10 +12,12 @@ import _frozen_importlib_external except ImportError: _frozen_importlib_external = _bootstrap_external -from ._abc import Loader import abc +# Public API +from ._abc import Loader + __all__ = [ 'Loader', 'MetaPathFinder', 'PathEntryFinder', 'ResourceLoader', 'InspectLoader', 'ExecutionLoader', diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py index 023f77d750fd2bc..b22e538862299ff 100644 --- a/Lib/importlib/machinery.py +++ b/Lib/importlib/machinery.py @@ -1,5 +1,9 @@ """The machinery of importlib: finders, loaders, hooks, etc.""" +lazy import warnings + + +# Public API from ._bootstrap import ModuleSpec from ._bootstrap import BuiltinImporter from ._bootstrap import FrozenImporter @@ -33,8 +37,6 @@ def all_suffixes(): def __getattr__(name): - import warnings - if name == 'DEBUG_BYTECODE_SUFFIXES': warnings.warn('importlib.machinery.DEBUG_BYTECODE_SUFFIXES is ' 'deprecated; use importlib.machinery.BYTECODE_SUFFIXES ' diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 2b564e9b52e0cb2..dbce696dc8d169a 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -1,19 +1,23 @@ """Utility code for constructing importers, etc.""" + +from ._bootstrap import _resolve_name +from ._bootstrap import _find_spec + +import _imp +import sys +import types + + +# Public API from ._abc import Loader from ._bootstrap import module_from_spec -from ._bootstrap import _resolve_name from ._bootstrap import spec_from_loader -from ._bootstrap import _find_spec from ._bootstrap_external import MAGIC_NUMBER from ._bootstrap_external import cache_from_source from ._bootstrap_external import decode_source from ._bootstrap_external import source_from_cache from ._bootstrap_external import spec_from_file_location -import _imp -import sys -import types - def source_hash(source_bytes): "Return the hash of *source_bytes* as used in hash-based pyc files." diff --git a/Lib/inspect.py b/Lib/inspect.py index af6aa3eb37a53bb..2a14e43b66f2fac 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -196,18 +196,18 @@ def ispackage(object): def ismethoddescriptor(object): """Return true if the object is a method descriptor. - But not if ismethod() or isclass() or isfunction() are true. + But not if ismethod(), isclass() or isfunction() is true. - This is new in Python 2.2, and, for example, is true of int.__add__. - An object passing this test has a __get__ attribute, but not a - __set__ attribute or a __delete__ attribute. Beyond that, the set - of attributes varies; __name__ is usually sensible, and __doc__ - often is. + An object passing this test (for example, int.__add__) has a __get__ + attribute, but not a __set__ attribute or a __delete__ attribute. + Beyond that, the set of attributes varies; __name__ is usually + sensible, and __doc__ often is. Methods implemented via descriptors that also pass one of the other - tests return false from the ismethoddescriptor() test, simply because - the other tests promise more -- you can, e.g., count on having the - __func__ attribute (etc) when an object passes ismethod().""" + tests (ismethod(), isclass(), isfunction()) make this function return + false, simply because those other tests promise more -- you can, for + example, count on having the __func__ attribute when an object passes + ismethod().""" if isclass(object) or ismethod(object) or isfunction(object): # mutual exclusion return False @@ -219,8 +219,13 @@ def ismethoddescriptor(object): def isdatadescriptor(object): """Return true if the object is a data descriptor. + But not if ismethod(), isclass() or isfunction() is true. + Data descriptors have a __set__ or a __delete__ attribute. Examples are - properties (defined in Python) and getsets and members (defined in C). + properties, getsets, and members. For the latter two (defined only in C + extension modules) more specific tests are available as well: + isgetsetdescriptor() and ismemberdescriptor(), respectively. + Typically, data descriptors will also have __name__ and __doc__ attributes (properties, getsets, and members have both of these attributes), but this is not guaranteed.""" diff --git a/Lib/logging/config.py b/Lib/logging/config.py index 35233e731eb42d1..fab91e663a0f6aa 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -32,6 +32,7 @@ import os import queue import re +import socket import struct import threading import traceback @@ -1004,6 +1005,15 @@ class ConfigSocketReceiver(ThreadingTCPServer): def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): + # The host can have no IPv4 address, for example if "localhost" + # is only aliased to ::1. Leave resolution errors to the server. + try: + infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except OSError: + pass + else: + if not any(info[0] == socket.AF_INET for info in infos): + self.address_family = infos[0][0] ThreadingTCPServer.__init__(self, (host, port), handler) with logging._lock: self.abort = 0 @@ -1035,9 +1045,14 @@ def __init__(self, rcvr, hdlr, port, verify): self.ready = threading.Event() def run(self): - server = self.rcvr(port=self.port, handler=self.hdlr, - ready=self.ready, - verify=self.verify) + try: + server = self.rcvr(port=self.port, handler=self.hdlr, + ready=self.ready, + verify=self.verify) + except BaseException: + # Do not leave the caller waiting for ready forever. + self.ready.set() + raise if self.port == 0: self.port = server.server_address[1] self.ready.set() diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index a5394d2dbea6494..c78a30763d9fa08 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -886,8 +886,9 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), """ Initialize a handler. - If address is specified as a string, a UNIX socket is used. To log to a - local syslogd, "SysLogHandler(address="/dev/log")" can be used. + If address is specified as a string or bytes, a UNIX socket is used. + To log to a local syslogd, "SysLogHandler(address="/dev/log")" can be + used. If facility is not specified, LOG_USER is used. If socktype is specified as socket.SOCK_DGRAM or socket.SOCK_STREAM, that specific socket type will be used. For Unix sockets, you can also specify a @@ -938,7 +939,7 @@ def createSocket(self): address = self.address socktype = self.socktype - if isinstance(address, str): + if not isinstance(address, (list, tuple)): self.unixsocket = True # Syslog server may be unavailable during handler initialisation. # C's openlog() function also ignores connection errors. diff --git a/Lib/pdb.py b/Lib/pdb.py index 01451f0229cacb2..458eb8352652366 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -3158,6 +3158,15 @@ def postloop(self): if self.quitting: self.detach() + @contextmanager + def _maybe_use_pyrepl_as_stdin(self): + # The server reads every command from the client over the socket, never + # from a local pyrepl. The base implementation swaps in pyrepl as stdin + # and blanks `self.prompt` to '' (pyrepl would draw the prompt itself), + # which here would transmit an empty prompt to the client whenever the + # target process happens to be pyrepl-capable. Keep the real prompt. + yield + def detach(self): # Detach the debugger and close the socket without raising BdbQuit self.quitting = False diff --git a/Lib/platform.py b/Lib/platform.py index 36489d4fdd98ae2..3141998e02dfe21 100644 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -543,21 +543,25 @@ def android_ver(release="", api_level=0, manufacturer="", model="", device="", is_emulator=False): if sys.platform == "android": try: - from ctypes import CDLL, c_char_p, create_string_buffer + from ctypes import CDLL, c_int, c_char_p, create_string_buffer + from ctypes.util import wrap_dll_function except ImportError: pass else: # An NDK developer confirmed that this is an officially-supported # API (https://stackoverflow.com/a/28416743). Use `getattr` to avoid # private name mangling. - system_property_get = getattr(CDLL("libc.so"), "__system_property_get") - system_property_get.argtypes = (c_char_p, c_char_p) + libc = CDLL("libc.so") + + @wrap_dll_function(libc) + def __system_property_get(name: c_char_p, value: c_char_p) -> c_int: + pass def getprop(name, default): # https://android.googlesource.com/platform/bionic/+/refs/tags/android-5.0.0_r1/libc/include/sys/system_properties.h#39 PROP_VALUE_MAX = 92 buffer = create_string_buffer(PROP_VALUE_MAX) - length = system_property_get(name.encode("UTF-8"), buffer) + length = __system_property_get(name.encode("UTF-8"), buffer) if length == 0: # This API doesn’t distinguish between an empty property and # a missing one. diff --git a/Lib/profiling/sampling/cli.py b/Lib/profiling/sampling/cli.py index 466b0aceae2dcc6..b47b166c6cd9408 100644 --- a/Lib/profiling/sampling/cli.py +++ b/Lib/profiling/sampling/cli.py @@ -87,8 +87,8 @@ def __call__(self, parser, namespace, values, option_string=None): # Constants for socket synchronization -_SYNC_TIMEOUT_SEC = 5.0 -_PROCESS_KILL_TIMEOUT_SEC = 2.0 +_SYNC_TIMEOUT_SEC = 15.0 +_PROCESS_KILL_TIMEOUT_SEC = 5.0 _READY_MESSAGE = b"ready" _RECV_BUFFER_SIZE = 1024 _BINARY_PROFILE_HEADER_SIZE = 64 diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 041ee26c791f36b..72974af26bee64c 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -620,55 +620,41 @@ def page(self, title, contents): def heading(self, title, extras=''): """Format a page heading.""" return ''' - - - -
 
%s		%s
- ''' % (title, extras or ' ') - - def section(self, title, cls, contents, width=6, - prelude='', marginalia=None, gap=' '): - """Format a section with a heading.""" - if marginalia is None: - marginalia = '' + ' ' * width + '' - result = '''

- - - - ''' % (cls, title) +
+

%s

+
%s
+
+''' % (title, extras) + + def _section(self, title, cls, contents, prelude, tag): + result = ''' +
+<%s>%s +''' % (cls, tag, title, tag) if prelude: - result = result + ''' -
- -''' % (cls, marginalia, cls, prelude, gap) - else: - result = result + ''' -''' % (cls, marginalia, gap) + result = result + '
%s
\n' % prelude + return result + '%s\n\n' % contents + + def section(self, title, cls, contents, width=None, + prelude='', marginalia=None, gap=None): + """Format a section with a heading. - return result + '\n
 
%s
%s%s
%s
%s%s%s
' % contents + The width, marginalia and gap arguments are ignored. + """ + return self._section(title, cls, contents, prelude, 'h3') - def bigsection(self, title, *args): + def bigsection(self, title, cls, contents, *ignored): """Format a section with a big heading.""" - title = '%s' % title - return self.section(title, *args) + return self._section(title, cls, contents, '', 'h2') def preformat(self, text): """Format literal preformatted text.""" - text = self.escape(text.expandtabs()) - return replace(text, '\n\n', '\n \n', '\n\n', '\n \n', - ' ', ' ', '\n', '
\n') + return self.escape(text.expandtabs()) def multicolumn(self, list, format): """Format a list of items into a multi-column list.""" - result = '' - rows = (len(list) + 3) // 4 - for col in range(4): - result = result + '' - for i in range(rows*col, rows*col+rows): - if i < len(list): - result = result + format(list[i]) + '
\n' - result = result + '' - return '%s
' % result + result = ''.join('

  • %s
    • \n' % format(item) for item in list) + return '
      \n%s
    ' % result def grey(self, text): return '%s' % text @@ -776,7 +762,7 @@ def formattree(self, tree, modname, parent=None): for entry in tree: if isinstance(entry, tuple): c, bases = entry - result = result + '
    ' + result = result + '
    ' result = result + self.classlink(c, modname) if bases and bases != (parent,): parents = [] @@ -787,7 +773,7 @@ def formattree(self, tree, modname, parent=None): elif isinstance(entry, list): result = result + '
    \n%s
    \n' % self.formattree( entry, modname, c) - return '
    \n%s
    \n' % result + return '
    \n%s
    \n' % result def docmodule(self, object, name=None, mod=None, *ignored): """Produce HTML documentation for a module object.""" @@ -800,10 +786,10 @@ def docmodule(self, object, name=None, mod=None, *ignored): links = [] for i in range(len(parts)-1): links.append( - '%s' % + '%s' % ('.'.join(parts[:i+1]), parts[i])) linkedname = '.'.join(links + parts[-1:]) - head = '%s' % linkedname + head = linkedname try: path = inspect.getabsfile(object) url = urllib.parse.quote(path) @@ -861,8 +847,8 @@ def docmodule(self, object, name=None, mod=None, *ignored): data.append((key, value)) doc = self.markup(getdoc(object), self.preformat, fdict, cdict) - doc = doc and '%s' % doc - result = result + '

    %s

    \n' % doc + doc = doc and '
    %s
    \n' % doc + result = result + doc if hasattr(object, '__path__'): modpkgs = [] @@ -895,9 +881,10 @@ def docmodule(self, object, name=None, mod=None, *ignored): if data: contents = [] for key, value in data: - contents.append(self.document(value, key)) + contents.append( + f'
    {self.document(value, key)}
    ') result = result + self.bigsection( - 'Data', 'data', '
    \n'.join(contents)) + 'Data', 'data', '\n'.join(contents)) if hasattr(object, '__author__'): contents = self.markup(str(object.__author__), self.preformat) result = result + self.bigsection('Author', 'author', contents) @@ -937,11 +924,26 @@ def maybe(self): object.__module__)) push('\n') + # Wrap the groups of members inherited from other classes in + #
    so that they are collapsed by default. + is_inherited = False + + def begingroup(msg): + if is_inherited: + push('
    \n' + '%s\n' % msg) + else: + push('

    %s

    \n' % msg) + + def endgroup(): + if is_inherited: + push('
    \n') + def spill(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() - push(msg) + begingroup(msg) for name, kind, homecls, value in ok: try: value = getattr(object, name) @@ -953,33 +955,37 @@ def spill(msg, attrs, predicate): push(self.document(value, name, mod, funcs, classes, mdict, object, homecls)) push('\n') + endgroup() return attrs def spilldescriptors(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() - push(msg) + begingroup(msg) for name, kind, homecls, value in ok: push(self.docdata(value, name, mod)) + endgroup() return attrs def spilldata(msg, attrs, predicate): ok, attrs = _split_list(attrs, predicate) if ok: hr.maybe() - push(msg) + begingroup(msg) for name, kind, homecls, value in ok: base = self.docother(getattr(object, name), name, mod) doc = getdoc(value) if not doc: - push('
    %s
    \n' % base) + push('
    %s
    \n' % base) else: doc = self.markup(getdoc(value), self.preformat, funcs, classes, mdict) - doc = '
    %s' % doc - push('
    %s%s
    \n' % (base, doc)) + push('
    %s
    ' + '
    %s
    \n' + % (base, doc)) push('\n') + endgroup() return attrs attrs = [(name, kind, cls, value) @@ -1014,10 +1020,12 @@ def spilldata(msg, attrs, predicate): continue elif thisclass is object: tag = 'defined here' + is_inherited = False else: tag = 'inherited from %s' % self.classlink(thisclass, object.__module__) - tag += ':
    \n' + is_inherited = True + tag += ':' sort_attributes(attrs, object) @@ -1040,10 +1048,10 @@ def spilldata(msg, attrs, predicate): contents = ''.join(contents) if name == realname: - title = 'class %s' % ( + title = 'class %s' % ( name, realname) else: - title = '%s = class %s' % ( + title = '%s = class %s' % ( name, name, realname) if bases: parents = [] @@ -1060,9 +1068,8 @@ def spilldata(msg, attrs, predicate): if decl: doc = decl + (doc or '') doc = self.markup(doc, self.preformat, funcs, classes, mdict) - doc = doc and '%s
         ' % doc - return self.section(title, 'title', contents, 3, doc) + return self.section(title, 'title', contents, prelude=doc) def formatvalue(self, object): """Format an argument default value as text.""" @@ -1115,7 +1122,7 @@ def docroutine(self, object, name=None, mod=None, asyncqualifier = '' if name == realname: - title = '%s' % (anchor, realname) + title = '%s' % (anchor, realname) else: if (cl is not None and inspect.getattr_static(cl, realname, []) is object): @@ -1126,7 +1133,7 @@ def docroutine(self, object, name=None, mod=None, note = '' else: reallink = realname - title = '%s = %s' % ( + title = '%s = %s' % ( anchor, name, reallink) argspec = None if inspect.isroutine(object): @@ -1142,15 +1149,15 @@ def docroutine(self, object, name=None, mod=None, argspec = '(...)' decl = asyncqualifier + title + self.escape(argspec) + (note and - self.grey('%s' % note)) + '%s' % note) if skipdocs: - return '
    %s
    \n' % decl + return '
    %s
    \n' % decl else: doc = self.markup( getdoc(object), self.preformat, funcs, classes, methods) - doc = doc and '
    %s
    ' % doc - return '
    %s
    %s
    \n' % (decl, doc) + doc = doc and '
    %s
    ' % doc + return '
    %s
    %s
    \n' % (decl, doc) def docdata(self, object, name=None, mod=None, cl=None, *ignored): """Produce html documentation for a data descriptor.""" @@ -1158,10 +1165,10 @@ def docdata(self, object, name=None, mod=None, cl=None, *ignored): push = results.append if name: - push('
    %s
    \n' % name) + push('
    %s
    \n' % name) doc = self.markup(getdoc(object), self.preformat) if doc: - push('
    %s
    \n' % doc) + push('
    %s
    \n' % doc) push('
    \n') return ''.join(results) @@ -2471,12 +2478,17 @@ def page(self, title, contents): '' % css_path) return '''\ - + + Pydoc: %s -%s%s
    %s
    +%s +%s +
    +%s +
    ''' % (title, css_link, html_navbar(), contents) @@ -2487,27 +2499,25 @@ def html_navbar(): platform.python_build()[0], platform.python_compiler())) return """ -
    - Python %s
    %s -
    -
    - -
    -
    - - -
      -
    - - -
    -
    -
    - """ % (version, html.escape(platform.platform(terse=True))) + +""" % (version, html.escape(platform.platform(terse=True))) def html_index(): """Module Index page.""" @@ -2515,13 +2525,11 @@ def html_index(): def bltinlink(name): return '%s' % (name, name) - heading = html.heading( - 'Index of Modules' - ) + heading = html.heading('Index of Modules') names = [name for name in sys.builtin_module_names if name != '__main__'] contents = html.multicolumn(names, bltinlink) - contents = [heading, '

    ' + html.bigsection( + contents = [heading, html.bigsection( 'Built-in Modules', 'index', contents)] seen = {} @@ -2529,8 +2537,8 @@ def bltinlink(name): contents.append(html.index(dir, seen)) contents.append( - '

    pydoc by Ka-Ping Yee' - '<ping@lfw.org>

    ') + '
    pydoc by Ka-Ping Yee' + ' <ping@lfw.org>
    ') return 'Index of Modules', ''.join(contents) def html_search(key): @@ -2554,13 +2562,11 @@ def bltinlink(name): return '%s' % (name, name) results = [] - heading = html.heading( - 'Search Results', - ) + heading = html.heading('Search Results') for name, desc in search_result: - results.append(bltinlink(name) + desc) + results.append('
  • %s%s
    • ' % (bltinlink(name), desc)) contents = heading + html.bigsection( - 'key = %s' % key, 'index', '
    '.join(results)) + 'key = %s' % key, 'index', '
      \n%s\n
    ' % '\n'.join(results)) return 'Search Results', contents def html_topics(): @@ -2569,29 +2575,21 @@ def html_topics(): def bltinlink(name): return '%s' % (name, name) - heading = html.heading( - 'INDEX', - ) + heading = html.heading('Topics') names = sorted(Helper.topics.keys()) - contents = html.multicolumn(names, bltinlink) - contents = heading + html.bigsection( - 'Topics', 'index', contents) + contents = heading + html.multicolumn(names, bltinlink) return 'Topics', contents def html_keywords(): """Index of keywords.""" - heading = html.heading( - 'INDEX', - ) + heading = html.heading('Keywords') names = sorted(Helper.keywords.keys()) def bltinlink(name): return '%s' % (name, name) - contents = html.multicolumn(names, bltinlink) - contents = heading + html.bigsection( - 'Keywords', 'index', contents) + contents = heading + html.multicolumn(names, bltinlink) return 'Keywords', contents def html_topicpage(topic): @@ -2603,11 +2601,9 @@ def html_topicpage(topic): title = 'KEYWORD' else: title = 'TOPIC' - heading = html.heading( - '%s' % title, - ) - contents = '
    %s
    ' % html.markup(contents) - contents = html.bigsection(topic , 'index', contents) + heading = html.heading('%s %s' % (title.capitalize(), topic)) + contents = ('
    %s
    ' + % (title.lower(), html.markup(contents))) if xrefs: xrefs = sorted(xrefs.split()) @@ -2615,7 +2611,7 @@ def bltinlink(name): return '%s' % (name, name) xrefs = html.multicolumn(xrefs, bltinlink) - xrefs = html.section('Related help topics: ', 'index', xrefs) + xrefs = html.section('Related help topics', 'index', xrefs) return ('%s %s' % (title, topic), ''.join((heading, contents, xrefs))) @@ -2628,9 +2624,7 @@ def html_getobj(url): return title, content def html_error(url, exc): - heading = html.heading( - 'Error', - ) + heading = html.heading('Error') contents = '
    '.join(html.escape(line) for line in format_exception_only(type(exc), exc)) contents = heading + html.bigsection(url, 'error', contents) diff --git a/Lib/pydoc_data/_pydoc.css b/Lib/pydoc_data/_pydoc.css index a6aa2e4c1a021ef..9e5c5c2f6f320dc 100644 --- a/Lib/pydoc_data/_pydoc.css +++ b/Lib/pydoc_data/_pydoc.css @@ -3,110 +3,253 @@ Contents of this file are subject to change without notice. + The colors and fonts follow the python-docs-theme used by + docs.python.org. */ +:root { + color-scheme: light dark; + --text-color: #333; + --background-color: #fff; + --link-color: #0072aa; + --link-visited-color: #6363bb; + --link-hover-color: #00b0e4; + --muted-color: #707070; + --code-background-color: #eee; + --border-color: #ccc; + --target-background-color: #fbe54e; + --block-background-color: #eeffcc; + --block-border-color: #ac9; + --error-color: #ba2121; + --repr-color: #c040c0; +} + +@media (prefers-color-scheme: dark) { + :root { + --text-color: rgba(255, 255, 255, 0.87); + --background-color: #222; + --link-color: #77aaff; + --link-visited-color: #0099ee; + --link-hover-color: #00b0e4; + --muted-color: #999; + --code-background-color: #424242; + --border-color: #616161; + --target-background-color: #616161; + --block-background-color: #333; + --block-border-color: #616161; + --error-color: #f44c4e; + --repr-color: #d080d0; + } +} + body { - background-color: #f0f0f8; + font-family: -apple-system, BlinkMacSystemFont, avenir next, avenir, + segoe ui, helvetica neue, helvetica, Cantarell, Ubuntu, roboto, + noto, arial, sans-serif; + color: var(--text-color); + background-color: var(--background-color); + line-height: 1.4; + margin: 0; + padding: 0 1em 1em; } -table.heading tr { - background-color: #7799ee; +code, pre, .docstring, dl.doc > dt { + font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, + monospace; + font-size: 96.5%; } -.decor { - color: #ffffff; +a:link { + color: var(--link-color); } -.title-decor { - background-color: #ffc8d8; - color: #000000; +a:visited { + color: var(--link-visited-color); } -.pkg-content-decor { - background-color: #aa55cc; +a:hover { + color: var(--link-hover-color); } -.index-decor { - background-color: #ee77aa; +/* Page heading */ + +header.heading { + display: flex; + flex-wrap: wrap; + justify-content: space-between; + align-items: flex-end; + column-gap: 1em; + padding: 0.5em 0; + border-bottom: 1px solid var(--border-color); } -.functions-decor { - background-color: #eeaa77; +header.heading h1 { + margin: 0; + font-size: 1.8em; } -.data-decor { - background-color: #55aa55; +header.heading .extra { + text-align: right; + overflow-wrap: anywhere; } -.author-decor { - background-color: #7799ee; +/* Sections */ + +h2, h3, h4 { + margin: 0.6em 0 0.4em; } -.credits-decor { - background-color: #7799ee; +section > h2 { + border-bottom: 1px solid var(--border-color); + padding-bottom: 0.2em; + font-size: 1.4em; } -.error-decor { - background-color: #bb0000; +section > h3 { + font-size: 1.1em; } -.grey { - color: #909090; +section.error > h2 { + color: var(--error-color); } -.white { - color: #ffffff; +section section { + border: 1px solid var(--border-color); + border-radius: 3px; + padding: 0 0.8em 0.5em; + margin: 0.8em 0; } -.repr { - color: #c040c0; +/* Docstrings and other preformatted text */ + +.docstring { + white-space: pre-wrap; + margin: 0.3em 0; } -table.heading tr td.title { - vertical-align: bottom; +pre.topic, pre.keyword { + background-color: var(--block-background-color); + border: 1px solid var(--block-border-color); + border-radius: 3px; + padding: 0.5em 0.8em; + overflow-x: auto; } -table.heading tr td.extra { - vertical-align: bottom; - text-align: right; +/* Documented names (functions, methods, data) */ + +dl.doc { + margin: 0.5em 0; } -.heading-text { - font-family: helvetica, arial; +dl.doc > dt { + overflow-wrap: anywhere; } -.bigsection { - font-size: larger; +a:target { + background-color: var(--target-background-color); } -.title { - font-size: x-large; +dl.doc > dd { + margin: 0.2em 0 0.8em 2em; } -.code { - font-family: monospace; +.note { + font-family: -apple-system, BlinkMacSystemFont, avenir next, avenir, + segoe ui, helvetica neue, helvetica, Cantarell, Ubuntu, roboto, + noto, arial, sans-serif; + color: var(--muted-color); } -table { - width: 100%; - border-spacing : 0; - border-collapse : collapse; +.grey { + color: var(--muted-color); +} + +.repr { + color: var(--repr-color); +} + +hr { border: 0; + border-top: 1px solid var(--border-color); + margin: 0.8em 0; } -td { - padding: 2; +/* Inherited class members are collapsed by default. */ + +details.inherited > summary { + cursor: pointer; + margin: 0.6em 0 0.4em; } -td.section-title { - vertical-align: bottom; +/* Multi-column lists of modules, topics and keywords */ + +ul.multicolumn { + list-style-type: none; + column-width: 14em; + column-gap: 1em; + margin: 0.5em 0; + padding: 0; } -td.multicolumn { - width: 25%; - vertical-align: bottom; +ul.multicolumn > li { + overflow-wrap: anywhere; } -td.singlecolumn { - width: 100%; +/* Class hierarchy trees */ + +dl.tree, dl.tree dl { + margin: 0; +} + +dl.tree dd { + margin: 0 0 0 2em; +} + +/* Navigation bar of the pydoc server */ + +nav.navbar { + display: flex; + flex-wrap: wrap; + justify-content: space-between; + align-items: center; + column-gap: 1em; + row-gap: 0.3em; + padding: 0.5em 0; + border-bottom: 1px solid var(--border-color); + font-size: smaller; +} + +nav.navbar .navbar-version { + color: var(--muted-color); +} + +nav.navbar ul { + display: flex; + flex-wrap: wrap; + column-gap: 1em; + list-style-type: none; + margin: 0; + padding: 0; +} + +nav.navbar form { + display: inline-block; +} + +nav.navbar input { + font-family: inherit; + font-size: inherit; + color: inherit; + background-color: var(--background-color); + border: 1px solid #999; + border-radius: 3px; +} + +footer { + margin-top: 1em; + padding-top: 0.3em; + border-top: 1px solid var(--border-color); + color: var(--muted-color); + font-size: smaller; + text-align: right; } diff --git a/Lib/random.py b/Lib/random.py index 4541267bab866a4..7db761034509d37 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -861,7 +861,11 @@ def binomialvariate(self, n=1, p=0.5): u = random() u -= 0.5 us = 0.5 - _fabs(u) - k = _floor((2.0 * a / us + b) * u + c) + try: + k = _floor((2.0 * a / us + b) * u + c) + except ZeroDivisionError: + # Reject case where random() returned 0.0 + continue if k < 0 or k > n: continue v = random() diff --git a/Lib/socket.py b/Lib/socket.py index 03c3fe88f15cfe6..2a4f875f76b069c 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -900,7 +900,9 @@ def has_dualstack_ipv6(): try: with socket(AF_INET6, SOCK_STREAM) as sock: sock.setsockopt(IPPROTO_IPV6, IPV6_V6ONLY, 0) - return True + # On some platforms (e.g. DragonFly BSD) setting IPV6_V6ONLY to 0 + # silently has no effect, so check that it was actually cleared. + return sock.getsockopt(IPPROTO_IPV6, IPV6_V6ONLY) == 0 except error: return False diff --git a/Lib/stat.py b/Lib/stat.py index 214c7917b5e0489..76f02d7aa33d160 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -1,7 +1,4 @@ -"""Constants/functions for interpreting results of os.stat() and os.lstat(). - -Suggested usage: from stat import * -""" +"""Constants/functions for interpreting results of os.stat() and os.lstat().""" # Indices for stat struct members in the tuple returned by os.stat() diff --git a/Lib/tempfile.py b/Lib/tempfile.py index 485038417c65c66..b5f799f8d685545 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -929,7 +929,8 @@ def __init__(self, suffix=None, prefix=None, dir=None, @classmethod def _rmtree(cls, name, ignore_errors=False, repeated=False): def onexc(func, path, exc): - if isinstance(exc, PermissionError): + # On DragonFly BSD, UF_NOUNLINK removal fails with EISDIR, not EPERM. + if isinstance(exc, (PermissionError, IsADirectoryError)): if repeated and path == name: if ignore_errors: return diff --git a/Lib/test/_isolated_sample.py b/Lib/test/_isolated_sample.py new file mode 100644 index 000000000000000..360a27a2b081173 --- /dev/null +++ b/Lib/test/_isolated_sample.py @@ -0,0 +1,111 @@ +"""Sample tests driven by test.test_support.TestIsolated. + +This module is imported, never run as a test file, so that +:func:`test.support.isolation.runInSubprocess` has a real, importable target to run in +a subprocess. Several of these tests fail, error or are skipped on purpose. +""" + +import time +import unittest +from test.support import isolation + +# DurationSample sleeps this long in the subprocess; a parent-reported duration +# close to it proves the subprocess timing was forwarded, not the replay time. +DURATION_SLEEP = 0.2 + + +class MethodSample(unittest.TestCase): + + @isolation.runInSubprocess() + def test_pass(self): + self.assertTrue(isolation.runningInSubprocess) + + @isolation.runInSubprocess() + def test_fail(self): + self.assertEqual(1, 2) + + @isolation.runInSubprocess() + def test_error(self): + raise RuntimeError('boom') + + @isolation.runInSubprocess() + def test_skip(self): + self.skipTest('nope') + + @isolation.runInSubprocess() + @unittest.expectedFailure + def test_expected_failure(self): + self.assertEqual(1, 2) + + @isolation.runInSubprocess() + @unittest.expectedFailure + def test_unexpected_success(self): + pass + + +@isolation.runInSubprocess() +class ClassSample(unittest.TestCase): + + def test_pass(self): + self.assertTrue(isolation.runningInSubprocess) + + def test_fail(self): + self.assertEqual(1, 2) + + @unittest.expectedFailure + def test_expected_failure(self): + self.assertEqual(1, 2) + + +class SubtestSample(unittest.TestCase): + + @isolation.runInSubprocess() + def test_subtests(self): + for i in range(3): + with self.subTest(i=i): + self.assertNotEqual(i, 1) + + +@isolation.runInSubprocess() +class DurationSample(unittest.TestCase): + + def test_slow(self): + time.sleep(DURATION_SLEEP) + + +@isolation.runInSubprocess() +class SubclassingSample(unittest.TestCase): + # setUpClass must run bound to the runtime class, so a subclass sees its own + # name here rather than the base class's. + + @classmethod + def setUpClass(cls): + cls.setup_class_name = cls.__name__ + + def setUp(self): + self.set_up = True + + def test_runtime_class(self): + self.assertEqual(self.setup_class_name, type(self).__name__) + + +class SubclassSample(SubclassingSample): + # What a subclass adds or overrides must run in the subprocess too. + + def setUp(self): + super().setUp() + self.set_up_in_subclass = True + + def test_added_in_subclass(self): + self.assertTrue(isolation.runningInSubprocess) + self.assertTrue(self.set_up) + self.assertTrue(self.set_up_in_subclass) + + +class BrokenSubclassSample(SubclassingSample): + # An overriding setUpClass() that does not call super() bypasses the + # subprocess entirely. + + @classmethod + def setUpClass(cls): + pass diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 115a187a8a85882..8f665b3a98a0371 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -161,9 +161,10 @@ def wait_for_handle(handle, timeout): # try: - from ctypes import Structure, c_int, c_double, c_longlong + from ctypes.util import struct as ctypes_struct + from ctypes import c_int, c_double, c_longlong except ImportError: - Structure = object + def ctypes_struct(cls): return cls c_int = c_double = c_longlong = None @@ -4390,12 +4391,11 @@ def test_free_from_gc(self): # # -class _Foo(Structure): - _fields_ = [ - ('x', c_int), - ('y', c_double), - ('z', c_longlong,) - ] +@ctypes_struct +class _Foo: + x: c_int + y: c_double + z: c_longlong class _TestSharedCTypes(BaseTestCase): diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 8be5bf8aa4f5469..53bd369fabe21f7 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -109,6 +109,16 @@ def test_block_add_hook_baseexception(): pass +def test_block_add_hook_valueerror(): + # Non-RuntimeError exceptions (like ValueError) should propagate out + with assertRaises(ValueError): + with TestHook( + raise_on_events="sys.addaudithook", exc_type=ValueError + ) as hook1: + with TestHook() as hook2: + pass + + def test_marshal(): import marshal o = ("a", "b", "c", 1, 2, 3) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index befdac7ee77f107..7e6c7fa4cc5507a 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -147,9 +147,12 @@ def _kill(self) -> None: use_killpg = USE_PROCESS_GROUP if use_killpg: - parent_sid = os.getsid(0) - sid = os.getsid(popen.pid) - use_killpg = (sid != parent_sid) + try: + use_killpg = (os.getsid(popen.pid) != os.getsid(0)) + except PermissionError: + # On OpenBSD getsid() is only allowed for a process in the + # same session, so the failure means that it is not. + use_killpg = True if use_killpg: what = f"{self} process group" diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index b9b76a44e3b4e79..d62194acd9c29e5 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -50,6 +50,14 @@ def setup_process() -> None: for signum in signals: faulthandler.register(signum, chain=True, file=stderr_fd) + # Restore the default SIGINT handler if there is no Python-level + # handler. Python inherits the SIG_IGN disposition when the test + # suite runs as a shell background job; then asyncio.Runner does not + # install its own handler and _thread.interrupt_main() is a no-op, + # which makes some tests in test_asyncio and test_threading hang. + if signal.getsignal(signal.SIGINT) in (signal.SIG_IGN, signal.SIG_DFL): + signal.signal(signal.SIGINT, signal.default_int_handler) + adjust_rlimit_nofile() support.record_original_stdout(sys.stdout) diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index bacfe5e21ba0b7d..c7ec63763bc9d85 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -8,6 +8,7 @@ 'test_ctypes', 'test_concurrent_futures', 'test_enum', + 'test_enumerate', 'test_functools', 'test_httpservers', 'test_imaplib', diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index e76f79c274e744a..ec2aa59f2cb8728 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import ALWAYS_EQ, NEVER_EQ, skip_if_huge_c_stack from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow @@ -60,6 +60,7 @@ def test_repr(self): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") + @skip_if_huge_c_stack(200_000) @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_repr_deep(self): diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 9624072e69adfcc..ae2fb3f5f448e25 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -629,6 +629,7 @@ def __repr__(self): d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) + @support.skip_if_huge_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() @support.skip_if_sanitizer("requires deep stack", ub=True) diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 3227b91bd82a863..ea7edb798051567 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -570,6 +570,7 @@ def collect_sysconfig(info_add): for name in ( 'ABIFLAGS', + 'ALT_SOABI', 'ANDROID_API_LEVEL', 'CC', 'CCSHARED', @@ -595,6 +596,7 @@ def collect_sysconfig(info_add): 'Py_REMOTE_DEBUG', 'SHELL', 'SOABI', + 'SOABI_PLATFORM', 'TEST_MODULES', 'VAPTH', 'abs_builddir', @@ -995,7 +997,7 @@ def collect_windows(info_add): # windows.RtlAreLongPathsEnabled: RtlAreLongPathsEnabled() # windows.is_admin: IsUserAnAdmin() try: - import ctypes + import ctypes.util if not hasattr(ctypes, 'WinDLL'): raise ImportError except ImportError: @@ -1004,20 +1006,19 @@ def collect_windows(info_add): ntdll = ctypes.WinDLL('ntdll') BOOLEAN = ctypes.c_ubyte try: - RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled + @ctypes.util.wrap_dll_function(ntdll) + def RtlAreLongPathsEnabled() -> BOOLEAN: + pass except AttributeError: res = '' else: - RtlAreLongPathsEnabled.restype = BOOLEAN - RtlAreLongPathsEnabled.argtypes = () res = bool(RtlAreLongPathsEnabled()) info_add('windows.RtlAreLongPathsEnabled', res) - shell32 = ctypes.windll.shell32 - IsUserAnAdmin = shell32.IsUserAnAdmin - IsUserAnAdmin.restype = BOOLEAN - IsUserAnAdmin.argtypes = () - info_add('windows.is_admin', IsUserAnAdmin()) + @ctypes.util.wrap_dll_function(ctypes.windll.shell32) + def IsUserAnAdmin() -> BOOLEAN: + pass + info_add('windows.is_admin', bool(IsUserAnAdmin())) try: import _winapi @@ -1316,6 +1317,12 @@ def collect_system(info_add): info_add('system.hardware', hardware) +def collect_importlib(info_add): + import importlib.machinery + info_add('importlib.extension_suffixes', + importlib.machinery.EXTENSION_SUFFIXES) + + def collect_info(info): error = False info_add = info.add @@ -1358,6 +1365,7 @@ def collect_info(info): collect_zstd, collect_libregrtest_utils, collect_system, + collect_importlib, # Collecting from tests should be last as they have side effects. collect_test_socket, diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index d17d9a2ecf8d9b0..f4c4b4c1acfc182 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -45,7 +45,7 @@ "check__all__", "skip_if_buggy_ucrt_strfptime", "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer", "requires_limited_api", "requires_specialization", "thread_unsafe", - "skip_if_unlimited_stack_size", + "skip_if_unlimited_stack_size", "skip_if_huge_c_stack", # sys "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", @@ -236,20 +236,41 @@ def _is_gui_available(): # if Python is running as a service (such as the buildbot service), # gui interaction may be disallowed import ctypes + import ctypes.util import ctypes.wintypes + UOI_FLAGS = 1 WSF_VISIBLE = 0x0001 - class USEROBJECTFLAGS(ctypes.Structure): - _fields_ = [("fInherit", ctypes.wintypes.BOOL), - ("fReserved", ctypes.wintypes.BOOL), - ("dwFlags", ctypes.wintypes.DWORD)] - dll = ctypes.windll.user32 - h = dll.GetProcessWindowStation() + + @ctypes.util.struct + class USEROBJECTFLAGS: + fInherit: ctypes.wintypes.BOOL + fReserved: ctypes.wintypes.BOOL + dwFlags: ctypes.wintypes.DWORD + + user32 = ctypes.windll.user32 + + @ctypes.util.wrap_dll_function(user32) + def GetProcessWindowStation() -> ctypes.wintypes.HANDLE: + ... + + h = GetProcessWindowStation() if not h: raise ctypes.WinError() + + @ctypes.util.wrap_dll_function(user32) + def GetUserObjectInformationW( + hObj: ctypes.wintypes.HANDLE, + nIndex: ctypes.c_int, + pvInfo: ctypes.c_void_p, + nLength: ctypes.wintypes.DWORD, + lpnLengthNeeded: ctypes.POINTER(ctypes.wintypes.DWORD), + ) -> ctypes.wintypes.BOOL: + ... + uof = USEROBJECTFLAGS() needed = ctypes.wintypes.DWORD() - res = dll.GetUserObjectInformationW(h, + res = GetUserObjectInformationW(h, UOI_FLAGS, ctypes.byref(uof), ctypes.sizeof(uof), @@ -2818,6 +2839,32 @@ def exceeds_recursion_limit(): return 150_000 +def skip_if_huge_c_stack(depth=150_000): + """Skip decorator for tests which cannot overflow the C stack. + + Tests exhausting the C stack with *depth* recursive calls cannot + trigger the recursion protection if the C stack is too large (e.g. + with a large or unlimited RLIMIT_STACK), and either fail, or run + for a very long time, or crash, or consume all memory. + """ + try: + from _testinternalcapi import get_c_recursion_remaining + except ImportError: + # Fall back to checking for an unlimited stack size. + huge = False + if not (is_emscripten or is_wasi) and os.name != "nt": + import resource + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + huge = soft == hard and soft in (-1, 0xFFFF_FFFF_FFFF_FFFF) + else: + remaining = get_c_recursion_remaining() + # A negative value means integer overflow in the estimate + # (e.g. with an unlimited RLIMIT_STACK). + huge = remaining >= depth or remaining < 0 + return unittest.skipIf( + huge, f"the C stack is large enough for {depth} recursive calls") + + # Windows doesn't have os.uname() but it doesn't support s390x. is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x' skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x') diff --git a/Lib/test/support/isolation.py b/Lib/test/support/isolation.py new file mode 100644 index 000000000000000..f449bf44034da35 --- /dev/null +++ b/Lib/test/support/isolation.py @@ -0,0 +1,290 @@ +"""Run tests in isolated subprocesses (the test.support.isolation.runInSubprocess decorator). + +A failure, error or skip that happens in the subprocess is replayed in the +parent process so that the test runner records it. The original (subprocess) +traceback is attached as the cause of the replayed exception, the same way +:mod:`concurrent.futures` surfaces tracebacks from worker processes. +""" + +import functools +import os +import sys +import unittest + +# Let unittest strip this module's frames from tracebacks, so only the original +# subprocess traceback (attached as the cause) is shown, not the replay frames. +__unittest = True + +# test.support globals set by regrtest (libregrtest/setup.py) that affect how +# tests run and which are skipped at runtime in the subprocess. +_PROPAGATED_CONFIG = ( + 'use_resources', # -u (is_resource_enabled/requires) + 'max_memuse', 'real_max_memuse', # -M (bigmemtest) + 'verbose', # -v + 'failfast', # -f +) + +def _child_config(): + import test.support as support + return {name: getattr(support, name) for name in _PROPAGATED_CONFIG} + +def _apply_child_config(config): + """Set up the child to run the test like a regrtest worker would. + + Mark this process as the subprocess, mirror the parent's -u/-M/-v config, + then suppress the Windows CRT assertion dialogs, which would block a debug + build on a modal dialog and hang the parent. + """ + global runningInSubprocess + import marshal + import test.support as support + runningInSubprocess = True + for name, value in marshal.loads(bytes.fromhex(config)).items(): + setattr(support, name, value) + support.suppress_msvcrt_asserts(support.verbose >= 2) + +# True inside the subprocess spawned by @runInSubprocess(), set by +# _apply_child_config() before the test is imported. Fixtures can test it to +# decide what to run in the subprocess as opposed to the parent process. +runningInSubprocess = False + + +class _RemoteTraceback(Exception): + """Carry a formatted traceback string from the subprocess for display. + + Attached as the ``__cause__`` of the replayed failure/error, so that the + original traceback is shown by the traceback machinery. + """ + def __init__(self, tb): + self.tb = tb + + def __str__(self): + return self.tb + + +class _SubprocessTestError(Exception): + """Replay a subprocess error (as opposed to a failure) in the parent.""" + + +def _decode(data): + # Decode the child output, which is only ever shown as a diagnostic: an + # undecodable byte must not hide the failure it is part of. + if not data: + return '' + import locale + encoding = 'utf-8' if sys.flags.utf8_mode else locale.getencoding() + return data.decode(encoding, 'backslashreplace').replace('\r\n', '\n') + + +def _remote(detail): + # Wrap the subprocess traceback the way concurrent.futures does, so it is + # clearly delimited when shown as the cause. + return _RemoteTraceback(f'\n"""\n{detail}"""') + + +def _check_subprocess_support(): + # runInSubprocess() always runs the test in a subprocess, so skip (in the + # parent) on platforms that do not support spawning one. + import test.support as support + if not support.has_subprocess_support: + raise unittest.SkipTest('requires subprocess support') + + +def _run_in_subprocess(module, qualname): + """Run module.qualname (a test method or class) in a fresh subprocess. + + Return ``(payload, output, returncode)``, where *payload* is the decoded + ``{'outcomes': ..., 'durations': ...}`` mapping from the subprocess, or + ``None`` if it did not run to completion (crash, import error, ...). + """ + import marshal + import subprocess + import tempfile + fd, result_path = tempfile.mkstemp(suffix='.json') + os.close(fd) + try: + # Pass the config on the command line, not in the environment, so that + # the test cannot pass it on to the processes it spawns itself. Use + # marshal, not json: it is built in, so the child imports nothing that + # the test would not see in a normal test run. + cmd = [sys.executable, '-m', 'test.support.subprocess_runner', + module, qualname, result_path, + marshal.dumps(_child_config()).hex()] + proc = subprocess.run(cmd, capture_output=True) + try: + with open(result_path, 'rb') as f: + payload = marshal.load(f) + except (OSError, EOFError, ValueError): + payload = None + finally: + try: + os.unlink(result_path) + except OSError: + pass + return payload, _decode(proc.stdout) + _decode(proc.stderr), proc.returncode + + +def _replay_outcome(test, outcome): + kind = outcome['kind'] + detail = outcome['detail'] + if kind == 'skipped': + test.skipTest(detail) # the detail is the skip reason, not a traceback + elif kind in ('failure', 'expected_failure'): + # Replay an expected failure like a failure: the wrapper keeps the + # @expectedFailure marker (via functools.wraps), so the parent records + # the raised exception as an expectedFailure. + exc = test.failureException('test failed in the subprocess') + raise exc from _remote(detail) + else: # 'error' + exc = _SubprocessTestError('test failed in the subprocess') + raise exc from _remote(detail) + + +def _replay_outcomes(test, outcomes): + # Replay each subtest outcome in its own subTest() context so that they are + # reported individually, then replay the whole-test outcome (if any). + main = [] + for outcome in outcomes: + if outcome['subtest']: + with test.subTest(outcome['desc']): + _replay_outcome(test, outcome) + else: + main.append(outcome) + for outcome in main: + _replay_outcome(test, outcome) + + +def _raise_fixture_outcome(outcome): + # Reproduce a setUpClass()/setUpModule() failure or skip from the + # subprocess in a parent-process fixture, so it applies to every test. + if outcome['kind'] == 'skipped': + raise unittest.SkipTest(outcome['detail']) + exc = _SubprocessTestError('class failed in the subprocess') + raise exc from _remote(outcome['detail']) + + +def _isolate_method(func): + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + if runningInSubprocess: + # Already running in the subprocess: run the real test. + return func(self, *args, **kwargs) + _check_subprocess_support() + cls = type(self) + qualname = f'{cls.__qualname__}.{func.__name__}' + payload, output, returncode = _run_in_subprocess(cls.__module__, + qualname) + if payload is None: + exc = _SubprocessTestError( + f'test did not complete in a subprocess (exit code {returncode})') + raise exc from _remote(output) + # The parent measures this method's own duration (the real cost of the + # isolated run, subprocess startup included), so nothing to forward here. + _replay_outcomes(self, payload['outcomes']) + return wrapper + + +def _isolate_class(cls): + # Unwrap to the plain functions so the replacements can call them with the + # runtime cls; a bound classmethod would freeze the decoration-time class + # and a subclass would run the fixtures bound to the base class. + orig_setUpClass = cls.setUpClass.__func__ + orig_tearDownClass = cls.tearDownClass.__func__ + # Hook the _call*() indirections rather than setUp(), tearDown() and the + # test methods themselves, to cover what a subclass adds or overrides too. + orig_callSetUp = cls._callSetUp + orig_callTearDown = cls._callTearDown + orig_callTestMethod = cls._callTestMethod + orig_addDuration = cls._addDuration + + def setUpClass(cls): + if runningInSubprocess: + orig_setUpClass(cls) + return + _check_subprocess_support() + # Run the whole class in a single subprocess and stash the outcomes + # for the test methods to replay. + payload, output, returncode = _run_in_subprocess(cls.__module__, + cls.__qualname__) + if payload is None: + exc = _SubprocessTestError( + f'class did not complete in a subprocess (exit code {returncode})') + raise exc from _remote(output) + by_id = {} + for outcome in payload['outcomes']: + if outcome['fixture']: + # A setUpClass()/setUpModule() failure or skip: apply it to the + # whole class by raising it here, in the parent's setUpClass(). + _raise_fixture_outcome(outcome) + by_id.setdefault(outcome['id'], []).append(outcome) + cls._isolated_outcomes = by_id + cls._isolated_durations = dict(payload.get('durations', ())) + + def tearDownClass(cls): + if runningInSubprocess: + orig_tearDownClass(cls) + else: + cls._isolated_outcomes = None + cls._isolated_durations = None + + def _callSetUp(self): + # In the parent the real test does not run, so neither should setUp(). + if runningInSubprocess: + orig_callSetUp(self) + + def _callTearDown(self): + if runningInSubprocess: + orig_callTearDown(self) + + def _callTestMethod(self, method): + if runningInSubprocess: + orig_callTestMethod(self, method) + return + by_id = getattr(type(self), '_isolated_outcomes', None) + if by_id is None: + raise _SubprocessTestError( + f'{type(self).__name__} did not run in a subprocess; ' + f'an overriding setUpClass() must call super().setUpClass()') + _replay_outcomes(self, by_id.get(self.id(), [])) + + def _addDuration(self, result, elapsed): + # In the parent, report the subprocess timing rather than the (instant) + # replay time; subprocess startup is paid once, in setUpClass. + if not runningInSubprocess: + durations = getattr(type(self), '_isolated_durations', None) or {} + elapsed = durations.get(self.id(), elapsed) + orig_addDuration(self, result, elapsed) + + cls.setUpClass = classmethod(setUpClass) + cls.tearDownClass = classmethod(tearDownClass) + cls._callSetUp = _callSetUp + cls._callTearDown = _callTearDown + cls._callTestMethod = _callTestMethod + cls._addDuration = _addDuration + return cls + + +def runInSubprocess(): + """Decorator to run a test method or class in a fresh subprocess. + + The decorated test runs in a separate, fresh Python process, so it does not + share global or interpreter state with the rest of the test run. When a + :class:`~unittest.TestCase` subclass is decorated, the whole class runs in a + single subprocess and its ``setUpClass()``/``setUpModule()`` fixtures run + once there; when a method is decorated, only that method runs in a + subprocess. Decorated methods must take no extra arguments. + + A failure, error or skip of the whole test is reported for the test, and + individual subtests (:meth:`~unittest.TestCase.subTest`) that fail or are + skipped are reported individually. The original subprocess traceback is + shown as the cause of a reported failure or error. Use + :data:`runningInSubprocess` in fixtures to choose what to run in the subprocess. + + The test is skipped on platforms without subprocess support, since it must + spawn one. + """ + def decorator(obj): + if isinstance(obj, type) and issubclass(obj, unittest.TestCase): + return _isolate_class(obj) + return _isolate_method(obj) + return decorator diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 2c45fe2369ec36b..daf6060940e97f0 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -467,12 +467,23 @@ def _rmtree(path): def _rmtree_inner(path): from test.support import _force_run + # Clear file flags (e.g. UF_IMMUTABLE, UF_NOUNLINK on BSD). + if hasattr(os, 'chflags'): + try: + os.chflags(path, 0) + except OSError: + pass for name in _force_run(path, os.listdir, path): fullname = os.path.join(path, name) try: mode = os.lstat(fullname).st_mode except OSError: mode = 0 + if hasattr(os, 'lchflags'): + try: + os.lchflags(fullname, 0) + except OSError: + pass if stat.S_ISDIR(mode): _rmtree_inner(fullname) _force_run(path, os.rmdir, fullname) diff --git a/Lib/test/support/subprocess_runner.py b/Lib/test/support/subprocess_runner.py new file mode 100644 index 000000000000000..90d74cc757d878d --- /dev/null +++ b/Lib/test/support/subprocess_runner.py @@ -0,0 +1,79 @@ +"""Run a single test method in this (sub)process and report the result. + +Invoked as ``python -m test.support.subprocess_runner MODULE QUALNAME OUTFILE +CONFIG`` by :func:`test.support.isolation.runInSubprocess`. CONFIG is the +marshalled test.support configuration of the parent test run, as a hex string. +The outcome of the test (including that of each individual subtest) is +marshalled to OUTFILE. This module is not meant to be imported. + +Import as little as possible before running the test: every module imported +here is state that the test would not see in a normal test run. +""" + +import marshal +import sys +import unittest +from unittest.case import _SubTest + +if __name__ != '__main__': + raise ImportError('this module cannot be directly imported') + +if len(sys.argv) != 5: + print('usage: python -m test.support.subprocess_runner ' + 'MODULE QUALNAME OUTFILE CONFIG', file=sys.stderr) + sys.exit(2) + +module, qualname, outfile, config = sys.argv[1:] + +# Set up the child before importing the test. +from test.support.isolation import _apply_child_config +_apply_child_config(config) + + +class _Result(unittest.TestResult): + # Capture per-test durations keyed by test id, so the parent can report the + # subprocess timings instead of its own replay time. + def __init__(self): + super().__init__() + self.id_durations = [] + + def addDuration(self, test, elapsed): + super().addDuration(test, elapsed) + self.id_durations.append((test.id(), elapsed)) + + +# Resolve the qualname in the imported module, rather than letting +# loadTestsFromName() guess where the module name ends: it guesses by trying +# imports that fail, and a failing import pulls in importlib.resources. +__import__(module) +suite = unittest.TestLoader().loadTestsFromName(qualname, sys.modules[module]) +result = _Result() +suite.run(result) + + +def _outcome(kind, test, detail): + subtest = isinstance(test, _SubTest) + real = test.test_case if subtest else test + return { + 'kind': kind, + 'subtest': subtest, + 'desc': test._subDescription() if subtest else '', + # id() groups outcomes by test method; a non-TestCase (e.g. an + # _ErrorHolder) marks a setUpClass()/setUpModule() fixture failure. + 'id': real.id(), + 'fixture': not isinstance(real, unittest.TestCase), + 'detail': detail, + } + + +outcomes = [_outcome('failure', t, tb) for t, tb in result.failures] +outcomes += [_outcome('error', t, tb) for t, tb in result.errors] +outcomes += [_outcome('expected_failure', t, tb) + for t, tb in result.expectedFailures] +outcomes += [_outcome('skipped', t, reason) for t, reason in result.skipped] + +payload = {'outcomes': outcomes, 'durations': result.id_durations} +with open(outfile, 'wb') as f: + marshal.dump(payload, f) + +sys.exit(0) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index 31daafbc3d63009..201d701aff11eb2 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -41,13 +41,18 @@ def logcat_thread(): try: from ctypes import CDLL, c_char_p, c_int - android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") - android_log_write.argtypes = (c_int, c_char_p, c_char_p) - ANDROID_LOG_INFO = 4 + from ctypes.util import wrap_dll_function + liblog = CDLL("liblog.so") + + @wrap_dll_function(liblog) + def __android_log_write(prio: c_int, tag: c_char_p, + text: c_char_p) -> c_int: + pass # Separate tests using a marker line with a different tag. + ANDROID_LOG_INFO = 4 tag, message = "python.test", f"{self.id()} {time()}" - android_log_write( + __android_log_write( ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8")) self.assert_log("I", tag, message, skip=True) except: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 1dc3f538f4ad8ba..442cbb1aaadfe3f 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -3,7 +3,6 @@ import _colorize import contextlib import functools -import inspect import io import operator import os @@ -12,6 +11,7 @@ import sys import textwrap import tempfile +import types import unittest import argparse import warnings @@ -85,6 +85,8 @@ class TestLazyImports(unittest.TestCase): "_colorize", "copy", "difflib", + "gettext", + "re", "shutil", "textwrap", "warnings", @@ -99,7 +101,7 @@ def test_create_parser(self): # Test imports are still unused after # creating a parser create_parser = "argparse.ArgumentParser()" - imported_modules = {"shutil"} + imported_modules = {"gettext", "re", "shutil"} import_helper.ensure_lazy_imports( "argparse", @@ -114,7 +116,7 @@ def test_add_subparser(self): parser.add_subparsers(dest='command', required=False) """ ) - imported_modules = {"shutil"} + imported_modules = {"gettext", "re", "shutil"} import_helper.ensure_lazy_imports( "argparse", @@ -132,7 +134,7 @@ def test_parse_args(self): parser.parse_args(['BAR', '--foo', 'FOO']) """ ) - imported_modules = {"shutil"} + imported_modules = {"gettext", "re", "shutil"} import_helper.ensure_lazy_imports( "argparse", self.LAZY_IMPORTS - imported_modules, @@ -7098,7 +7100,10 @@ def test_all_exports_everything_but_modules(self): name for name, value in vars(argparse).items() if not (name.startswith("_") or name == 'ngettext') - if not inspect.ismodule(value) + if not isinstance( + value, + (types.ModuleType, types.LazyImportType), + ) ] self.assertEqual(sorted(items), sorted(argparse.__all__)) @@ -7580,6 +7585,62 @@ def test_argparse_color_custom_usage(self): ), ) + def test_argparse_color_wrapping_matches_uncolored(self): + # gh-142035: color codes must not affect where help text wraps. + # Stripping the escapes from colored help must yield exactly the + # same text as the uncolored help across representative widths. + def build(color, path="output.txt"): + parser = argparse.ArgumentParser(prog="PROG", color=color) + parser.add_argument( + "--mode", + default="auto", + choices=("auto", "fast", "slow"), + help="select the operating mode from the available choices " + "%(choices)s and note the default is %(default)s here", + ) + parser.add_argument( + "--path", + default=path, + help="write output to %(default)s and continue processing", + ) + return parser + + env = self.enterContext(os_helper.EnvironmentVarGuard()) + paths = ( + "output.txt", + "/var/lib/application/cache/unusually_long_generated_filename", + "production-read-only-replica", + ) + for path in paths: + for columns in ("80", "60", "45", "30", "20"): + with self.subTest(path=path, columns=columns): + env["COLUMNS"] = columns + colored = build(color=True, path=path).format_help() + plain = build(color=False, path=path).format_help() + self.assertIn( + f"{self.theme.interpolated_value}auto" + f"{self.theme.reset}", + colored, + ) + self.assertEqual(_colorize.decolor(colored), plain) + + def test_argparse_color_preserved_when_wrapping_between_words(self): + parser = argparse.ArgumentParser(prog="PROG", color=True) + parser.add_argument( + "--mode", default="auto", + help="select the %(default)s operating mode from the available " + "options and continue with several more words", + ) + + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env["COLUMNS"] = "40" + help_text = parser.format_help() + + self.assertIn( + f"{self.theme.interpolated_value}auto{self.theme.reset}", + help_text, + ) + def test_custom_formatter_function(self): def custom_formatter(prog): return argparse.RawTextHelpFormatter(prog, indent_increment=5) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 31e218df127ae6b..87d63fcd8529336 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -26,7 +26,9 @@ from test import support from test.support import os_helper -from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, skip_if_unlimited_stack_size +from test.support import (skip_emscripten_stack_overflow, + skip_wasi_stack_overflow, + skip_if_unlimited_stack_size, skip_if_huge_c_stack) from test.support.ast_helper import ASTTestMixin from test.support.import_helper import ensure_lazy_imports from test.test_ast.utils import to_tuple @@ -1023,7 +1025,7 @@ def next(self): enum._test_simple_enum(_Precedence, _ast_unparse._Precedence) @support.cpython_only - @skip_if_unlimited_stack_size + @support.skip_if_huge_c_stack(100_000 if sys.platform == "android" else 500_000) @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_ast_recursion_limit(self): @@ -2096,7 +2098,7 @@ def test_level_as_none(self): exec(code, ns) self.assertIn('sleep', ns) - @skip_if_unlimited_stack_size + @skip_if_huge_c_stack() @skip_emscripten_stack_overflow() def test_recursion_direct(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) @@ -2105,7 +2107,7 @@ def test_recursion_direct(self): with support.infinite_recursion(): compile(ast.Expression(e), "", "eval") - @skip_if_unlimited_stack_size + @skip_if_huge_c_stack() @skip_emscripten_stack_overflow() def test_recursion_indirect(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 1622f19f0213f3c..18afdca23163a1e 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -1042,6 +1042,60 @@ async def iter_one(): asyncio.create_task(iter_one()) return status + def test_shutdown_asyncgens_reports_base_exceptions(self): + # gh-150866: shutdown_asyncgens silently swallowed exceptions that + # don't inherit from Exception raised during aclose() because the + # check was isinstance(result, Exception), but CancelledError inherits + # from BaseException. + self.loop._process_events = mock.Mock() + self.loop._write_to_self = mock.Mock() + + class MyBaseException(BaseException): + pass + + async def agen_cancel(): + try: + yield 1 + finally: + raise asyncio.CancelledError("agen got cancelled during cleanup") + + async def agen_base(): + try: + yield 1 + finally: + raise MyBaseException("base exc during cleanup") + + async def agen_value_error(): + try: + yield 1 + finally: + raise ValueError("agen failed during cleanup") + + caught = [] + + def handler(loop, context): + caught.append(context['exception']) + + async def main(): + loop = asyncio.get_running_loop() + loop.set_exception_handler(handler) + + g1 = agen_cancel() + g2 = agen_base() + g3 = agen_value_error() + await g1.__anext__() + await g2.__anext__() + await g3.__anext__() + + await loop.shutdown_asyncgens() + + self.loop.run_until_complete(main()) + self.assertEqual(len(caught), 3) + self.assertEqual( + {type(exc) for exc in caught}, + {asyncio.CancelledError, MyBaseException, ValueError}, + ) + def test_asyncgen_finalization_by_gc(self): # Async generators should be finalized when garbage collected. self.loop._process_events = mock.Mock() diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 91882f41d9428ed..3fdfffdb213efc2 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -1725,6 +1725,48 @@ def reader(data): self.loop.run_until_complete(proto.done) self.assertEqual('CLOSED', proto.state) + @unittest.skipUnless(sys.platform != 'win32', + "Don't support pipes for Windows") + @unittest.skipUnless(hasattr(os, 'mkfifo'), 'requires os.mkfifo()') + def test_write_named_fifo_unread_data(self): + # gh-145030: on macOS, the write end of a named FIFO polls as + # readable while unread data sits in the FIFO, which made the + # transport misinterpret the event as the reader hanging up + # and close itself. + path = os_helper.TESTFN + os.mkfifo(path) + self.assertNotEqual(os.stat(path).st_nlink, 0) + self.addCleanup(os_helper.unlink, path) + rfd = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + self.addCleanup(os.close, rfd) + wfd = os.open(path, os.O_WRONLY | os.O_NONBLOCK) + pipeobj = io.open(wfd, 'wb', 1024) + + proto = MyWritePipeProto(loop=self.loop) + connect = self.loop.connect_write_pipe(lambda: proto, pipeobj) + transport, p = self.loop.run_until_complete(connect) + self.assertIs(p, proto) + self.assertEqual('CONNECTED', proto.state) + + transport.write(b'1') + # Iterate the event loop while the data stays unread in the FIFO; + # the transport must not detect a false disconnection. + for _ in range(10): + test_utils.run_briefly(self.loop) + self.assertEqual('CONNECTED', proto.state) + self.assertFalse(transport.is_closing()) + self.assertEqual(b'1', os.read(rfd, 1024)) + + transport.write(b'2345') + for _ in range(10): + test_utils.run_briefly(self.loop) + self.assertEqual('CONNECTED', proto.state) + self.assertEqual(b'2345', os.read(rfd, 1024)) + + transport.close() + self.loop.run_until_complete(proto.done) + self.assertEqual('CLOSED', proto.state) + @unittest.skipUnless(sys.platform != 'win32', "Don't support pipes for Windows") def test_write_pipe_disconnect_on_close(self): diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 3dde7535284b255..928b618fe5c55b7 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -1,5 +1,6 @@ import asyncio import io +import sys import unittest from unittest import mock @@ -147,6 +148,31 @@ async def main(): 'async generator CallStackTestBase.test_stack_async_gen..gen()', stack_for_gen_nested_call[1]) + def test_ag_frame_used_for_async_generator(self): + # Regression test for gh-148736: the ag_await branch of + # _build_graph_for_future must read ag_frame, not cr_frame. + from asyncio.graph import _build_graph_for_future + + sentinel_frame = sys._getframe() + + class FakeAsyncGen: + ag_await = None + ag_frame = sentinel_frame + + class FakeCoro: + cr_frame = sentinel_frame + cr_await = FakeAsyncGen() + + loop = asyncio.new_event_loop() + try: + fut = loop.create_future() + fut.get_coro = lambda: FakeCoro() + result = _build_graph_for_future(fut) + finally: + loop.close() + + self.assertEqual(len(result.call_stack), 2) + async def test_stack_gather(self): stack_for_deep = None diff --git a/Lib/test/test_asyncio/test_protocols.py b/Lib/test/test_asyncio/test_protocols.py index 643199962b8af06..38f1e3fba90cfde 100644 --- a/Lib/test/test_asyncio/test_protocols.py +++ b/Lib/test/test_asyncio/test_protocols.py @@ -2,6 +2,7 @@ from unittest import mock import asyncio +from asyncio import protocols def tearDownModule(): @@ -63,5 +64,32 @@ def test_subprocess_protocol(self): self.assertNotHasAttr(sp, '__dict__') -if __name__ == '__main__': +class FeedDataToBufferedProtoTests(unittest.TestCase): + def _make_proto(self, bufsize): + received = bytearray() + buf = bytearray(bufsize) + + class P(asyncio.BufferedProtocol): + def get_buffer(self, sizehint): + return buf + + def buffer_updated(self, nbytes): + received.extend(buf[:nbytes]) + + return P(), received + + def test_large_multi_iteration(self): + proto, received = self._make_proto(64) + data = bytes(range(256)) * 16 + protocols._feed_data_to_buffered_proto(proto, data) + self.assertEqual(bytes(received), data) + + def test_memoryview_input(self): + proto, received = self._make_proto(64) + payload = b"y" * 200 + protocols._feed_data_to_buffered_proto(proto, memoryview(payload)) + self.assertEqual(bytes(received), payload) + + +if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index 69ac9a367f6be02..c8d429c3d1651f0 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -98,7 +98,12 @@ class SendfileBase: # 64 KiB page configuration. DATA = b"x" * (1024 * 17 * 64 + 1) # Reduce socket buffer size to test on relative small data sets. - BUF_SIZE = 4 * 1024 # 4 KiB + if sys.platform.startswith('dragonfly'): + # A smaller buffer makes every window update wait 100 ms for the + # delayed ACK timer. + BUF_SIZE = 32 * 1024 # 32 KiB + else: + BUF_SIZE = 4 * 1024 # 4 KiB def create_event_loop(self): raise NotImplementedError diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index bb1a7d6ff22be8d..f9b4fe53acb1343 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -12,7 +12,7 @@ from asyncio import subprocess from test.test_asyncio import utils as test_utils from test import support -from test.support import os_helper, warnings_helper, gc_collect +from test.support import os_helper, gc_collect if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") @@ -112,37 +112,6 @@ def test_subprocess_repr(self): ) transport.close() - def test_proc_exited_no_invalid_state_error_on_exit_waiters(self): - # gh-145541: when _connect_pipes hasn't completed (so - # _pipes_connected is False) and the process exits, _try_finish() - # sets the result on exit waiters. Then _call_connection_lost() must - # not call set_result() again on the same waiters. - self.loop.set_exception_handler( - lambda loop, context: self.fail( - f"unexpected exception: {context}") - ) - waiter = self.loop.create_future() - transport, protocol = self.create_transport(waiter) - - # Simulate a waiter registered via _wait() before the process exits. - exit_waiter = self.loop.create_future() - transport._exit_waiters.append(exit_waiter) - - # _connect_pipes hasn't completed, so _pipes_connected is False. - self.assertFalse(transport._pipes_connected) - - # Simulate process exit. _try_finish() will set the result on - # exit_waiter because _pipes_connected is False, and then schedule - # _call_connection_lost() because _pipes is empty (vacuously all - # disconnected). _call_connection_lost() must skip exit_waiter - # because it's already done. - transport._process_exited(6) - self.loop.run_until_complete(waiter) - - self.assertEqual(exit_waiter.result(), 6) - - transport.close() - class SubprocessMixin: @@ -208,6 +177,116 @@ async def run(): self.assertEqual(exitcode, 0) self.assertEqual(stdout, b'') + def test_communicate_cancelled_mid_read_retry(self): + # gh-139373: output read before communicate() was cancelled must + # be returned by a subsequent communicate() call. + code = ('import sys, time;' + 'sys.stdout.buffer.write(b"first\\n");' + 'sys.stdout.buffer.flush();' + 'time.sleep(3600)') + + async def run(): + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdout=subprocess.PIPE, + ) + task = asyncio.create_task(proc.communicate()) + # wait until communicate() has read the first line + while not proc._stdout_buf: + await asyncio.sleep(0.01) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + proc.kill() + stdout, stderr = await proc.communicate() + return stdout + + task = asyncio.wait_for(run(), support.LONG_TIMEOUT) + stdout = self.loop.run_until_complete(task) + self.assertEqual(stdout, b'first\n') + + def test_communicate_cancelled_during_wait_retry(self): + # gh-139373: cancellation landing after the output was fully read + # but before wait() completed must not lose the output. + code = ('import os, time;' + 'os.write(1, b"all output\\n");' + 'os.close(1);' + 'time.sleep(3600)') + + async def run(): + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdout=subprocess.PIPE, + ) + task = asyncio.create_task(proc.communicate()) + # wait until the stdout reader has consumed everything up to + # EOF; communicate() is then blocked on wait() + while not proc.stdout.at_eof(): + await asyncio.sleep(0.01) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + proc.kill() + stdout, stderr = await proc.communicate() + return stdout + + task = asyncio.wait_for(run(), support.LONG_TIMEOUT) + stdout = self.loop.run_until_complete(task) + self.assertEqual(stdout, b'all output\n') + + def test_communicate_cancelled_input_not_resendable(self): + # gh-139373: like subprocess.Popen.communicate(), sending new + # input after a cancelled communicate() call is an error. + async def run(): + proc = await asyncio.create_subprocess_exec( + *PROGRAM_CAT, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + task = asyncio.create_task(proc.communicate(b'data')) + await asyncio.sleep(0) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + with self.assertRaises(ValueError): + await proc.communicate(b'more data') + proc.kill() + await proc.communicate() + + self.loop.run_until_complete( + asyncio.wait_for(run(), support.LONG_TIMEOUT)) + + def test_communicate_cancelled_stdin_retry(self): + # gh-139373: input already fed before cancellation is not re-sent + # by the retried communicate() call, and the output is preserved. + code = ('import sys, time;' + 'sys.stdout.buffer.write(sys.stdin.buffer.read());' + 'sys.stdout.buffer.flush();' + 'time.sleep(3600)') + + async def run(): + proc = await asyncio.create_subprocess_exec( + sys.executable, '-c', code, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + task = asyncio.create_task(proc.communicate(b'hello')) + # the child echoes stdin only after it is closed, so once + # output arrives the input was fully written and + # communicate() is blocked on wait() + while not proc._stdout_buf: + await asyncio.sleep(0.01) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + proc.kill() + stdout, stderr = await proc.communicate() + return stdout + + task = asyncio.wait_for(run(), support.LONG_TIMEOUT) + stdout = self.loop.run_until_complete(task) + self.assertEqual(stdout, b'hello') + def test_shell(self): proc = self.loop.run_until_complete( asyncio.create_subprocess_shell('exit 7') @@ -251,7 +330,7 @@ def test_kill_issue43884(self): # kills the process and all its children. creationflags = CREATE_NEW_PROCESS_GROUP proc = self.loop.run_until_complete( - asyncio.create_subprocess_shell(blocking_shell_command, stdout=asyncio.subprocess.PIPE, + asyncio.create_subprocess_shell(blocking_shell_command, creationflags=creationflags) ) self.loop.run_until_complete(asyncio.sleep(1)) @@ -436,6 +515,46 @@ async def len_message(message): self.assertEqual(output.rstrip(), b'3') self.assertEqual(exitcode, 0) + def test_wait_even_if_pipe_is_open(self): + # gh-119710: Process.wait() must return once the process exits even + # if its stdout pipe is inherited by a grandchild that keeps it open, + # so the pipe never reaches EOF. Otherwise wait() hangs forever + # despite the returncode being known. + + async def run(): + # The grandchild inherits the child's stdin and stdout pipes and + # keeps both open after the child is killed. It writes "ready" + # so we know it has started, and exits once its stdin hits EOF. + code = textwrap.dedent("""\ + import subprocess, sys + subprocess.run([sys.executable, "-c", + "import sys; sys.stdout.write('ready');" + " sys.stdout.flush(); sys.stdin.read()"]) + """) + + proc = await asyncio.create_subprocess_exec( + sys.executable, "-c", code, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + try: + wait_proc = asyncio.create_task(proc.wait()) + # Wait until the grandchild holds the inherited pipes; this + # also lets the wait() task register its waiter. + await proc.stdout.readexactly(5) + proc.kill() + returncode = await asyncio.wait_for( + wait_proc, timeout=support.SHORT_TIMEOUT) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + else: + self.assertEqual(-signal.SIGKILL, returncode) + finally: + proc.stdin.close() # let the grandchild exit + await proc.stdout.read() + + self.loop.run_until_complete(run()) + def test_empty_input(self): async def empty_input(): @@ -909,7 +1028,6 @@ async def main(): self.loop.run_until_complete(main()) - @warnings_helper.ignore_warnings(category=ResourceWarning) def test_subprocess_read_pipe_cancelled(self): async def main(): loop = asyncio.get_running_loop() @@ -920,7 +1038,6 @@ async def main(): asyncio.run(main()) gc_collect() - @warnings_helper.ignore_warnings(category=ResourceWarning) def test_subprocess_write_pipe_cancelled(self): async def main(): loop = asyncio.get_running_loop() @@ -931,7 +1048,6 @@ async def main(): asyncio.run(main()) gc_collect() - @warnings_helper.ignore_warnings(category=ResourceWarning) def test_subprocess_read_write_pipe_cancelled(self): async def main(): loop = asyncio.get_running_loop() @@ -965,6 +1081,45 @@ def test_watcher_implementation(self): else: self.assertIsInstance(watcher, unix_events._ThreadedChildWatcher) + @unittest.skipUnless(hasattr(os, 'waitid'), 'needs os.waitid()') + def test_send_signal_never_targets_reaped_pid(self): + # gh-127049: there must be no window between the child watcher + # reaping the child and asyncio publishing the exit in which + # send_signal() signals the freed (possibly recycled) PID. + reaped_pids = set() + stale_kills = [] + orig_waitpid = os.waitpid + orig_kill = os.kill + + def waitpid(pid, options): + res = orig_waitpid(pid, options) + if res[0] != 0: + reaped_pids.add(res[0]) + return res + + def kill(pid, sig): + if pid in reaped_pids: + stale_kills.append(pid) + return + orig_kill(pid, sig) + + async def run(): + with mock.patch('os.waitpid', waitpid), \ + mock.patch('os.kill', kill): + proc = await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED) + proc.kill() + deadline = self.loop.time() + support.SHORT_TIMEOUT + while proc.returncode is None: + if self.loop.time() > deadline: + self.fail('child exit was not published in time') + proc.kill() + await asyncio.sleep(0) + await proc.wait() + self.assertEqual(stale_kills, []) + + self.loop.run_until_complete(run()) + class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): @@ -978,6 +1133,35 @@ def tearDown(self): unix_events.can_use_pidfd = self._original_can_use_pidfd return super().tearDown() + @unittest.skipUnless(hasattr(os, 'waitid'), 'needs os.waitid()') + def test_pid_not_reaped_before_exit_published(self): + # gh-127049: the watcher thread must wait for the child + # without reaping it: the PID must stay reserved until the + # event loop thread reaps it and publishes the exit as one + # atomic step. + async def run(): + proc = await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED) + thread = self.loop._watcher._threads.get(proc.pid) + self.assertIsNotNone(thread) + proc.kill() + # Wait for the watcher thread to observe the exit while + # the event loop cannot process the notification yet. + thread.join(support.SHORT_TIMEOUT) + self.assertFalse(thread.is_alive()) + # The exit has not been published yet... + self.assertIsNone(proc.returncode) + # ...so the child must still be an unreaped zombie and + # signalling its PID must still be safe. waitid() raises + # ChildProcessError if the PID was already reaped (and + # possibly recycled by the kernel). + os.waitid(os.P_PID, proc.pid, + os.WEXITED | os.WNOWAIT | os.WNOHANG) + proc.kill() + self.assertEqual(await proc.wait(), -signal.SIGKILL) + + self.loop.run_until_complete(run()) + @unittest.skipUnless( unix_events.can_use_pidfd(), "operating system does not support pidfds", diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 609ccdbfdb2b289..110cd0936c007c9 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2733,6 +2733,18 @@ async def main(): self.assertEqual(name, "example") await t + def test_eager_start_true_no_loop(self): + # gh-154695: eager_start must use the resolved loop, not loop=None. + async def asyncfn(): + return 42 + + async def main(): + t = self.__class__.Task(asyncfn(), eager_start=True) + self.assertTrue(t.done()) + self.assertEqual(await t, 42) + + asyncio.run(main(), loop_factory=asyncio.EventLoop) + def test_eager_start_false(self): name = None diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py index 5e743345028bec6..12f6e40f1595fde 100644 --- a/Lib/test/test_asyncio/test_transports.py +++ b/Lib/test/test_asyncio/test_transports.py @@ -98,6 +98,29 @@ def get_write_buffer_size(self): self.assertTrue(transport._protocol_paused) self.assertEqual(transport.get_write_buffer_limits(), (128, 256)) + def test_flowcontrol_mixin_pause_writing_exception(self): + + class MyTransport(transports._FlowControlMixin, + transports.Transport): + + def get_write_buffer_size(self): + return 2000 + + loop = mock.Mock() + transport = MyTransport(loop=loop) + protocol = mock.Mock() + protocol.pause_writing.side_effect = RuntimeError("boom") + transport._protocol = protocol + transport.set_write_buffer_limits(high=1000, low=100) + transport._maybe_pause_protocol() + protocol.pause_writing.assert_called_once() + loop.call_exception_handler.assert_called_once() + args = loop.call_exception_handler.call_args[0][0] + + self.assertIn("protocol.pause_writing() failed", args["message"]) + self.assertIsInstance(args["exception"], RuntimeError) + self.assertTrue(transport._protocol_paused) + def test_flowcontrol_mixin_compute_write_limits(self): class MyTransport(transports._FlowControlMixin, diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index db4e1eb9999c1fa..e9e546fdbfd6fbe 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -58,6 +58,9 @@ def test_block_add_hook(self): def test_block_add_hook_baseexception(self): self.do_test("test_block_add_hook_baseexception") + def test_block_add_hook_valueerror(self): + self.do_test("test_block_add_hook_valueerror") + def test_marshal(self): import_helper.import_module("marshal") diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 3213a4751273431..579e680448e94e3 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -37,6 +37,7 @@ try: import ctypes + import ctypes.util except ImportError: ctypes = None @@ -2849,8 +2850,11 @@ def test_memoryview_cast_1D_ND(self): if ctypes: # format: "T{>l:x:>d:y:}" - class BEPoint(ctypes.BigEndianStructure): - _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_double)] + @ctypes.util.struct(endian='big') + class BEPoint: + x: ctypes.c_long + y: ctypes.c_double + point = BEPoint(100, 200.1) m1 = memoryview(point) m2 = m1.cast('B') @@ -2873,6 +2877,32 @@ class BEPoint(ctypes.BigEndianStructure): self.assertEqual(m2.strides, (1,)) self.assertEqual(m2.suboffsets, ()) + def test_memoryview_cast_f_contiguous_ND_1D(self): + nd = ndarray(list(range(12)), shape=[3, 4], format='B', flags=ND_FORTRAN) + m = memoryview(nd) + self.assertTrue(m.f_contiguous) + self.assertTrue(m.contiguous) + + m1 = m.cast('B') + self.assertEqual(m1.ndim, 1) + self.assertEqual(m1.shape, (m.nbytes,)) + self.assertEqual(m1.strides, (1,)) + self.assertTrue(m1.c_contiguous) + self.assertTrue(m1.contiguous) + self.assertEqual(m1.tobytes(), memoryview(nd).tobytes(order='F')) + + for fmt in ('B', 'b', 'c', 'H', 'I'): + size = struct.calcsize(fmt) + if m.nbytes % size == 0: + m2 = m.cast(fmt) + self.assertEqual(m2.ndim, 1) + self.assertEqual(m2.shape, (m.nbytes // size,)) + self.assertTrue(m2.contiguous) + + m3 = m[::-1] + with self.assertRaises(TypeError): + m3.cast('B') + def test_memoryview_tolist(self): # Most tolist() tests are in self.verify() etc. @@ -3250,8 +3280,11 @@ def test_memoryview_compare_special_cases(self): # Some ctypes format strings are unknown to the struct module. if ctypes: # format: "T{>l:x:>l:y:}" - class BEPoint(ctypes.BigEndianStructure): - _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)] + @ctypes.util.struct(endian='big') + class BEPoint: + x: ctypes.c_long + y: ctypes.c_long + point = BEPoint(100, 200) a = memoryview(point) b = memoryview(point) @@ -3988,8 +4021,11 @@ def test_memoryview_tobytes(self): # Unknown formats are handled: tobytes() purely depends on itemsize. if ctypes: # format: "T{>l:x:>l:y:}" - class BEPoint(ctypes.BigEndianStructure): - _fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)] + @ctypes.util.struct(endian='big') + class BEPoint: + x: ctypes.c_long + y: ctypes.c_long + point = BEPoint(100, 200) a = memoryview(point) self.assertEqual(a.tobytes(), bytes(point)) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index e211c3d15a4ed20..720b38cb508cbe6 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1637,6 +1637,35 @@ def test_take_bytes_optimization(self): bytes_header_size = sys.getsizeof(b'') self.assertEqual(ba.__alloc__(), 499 + bytes_header_size) + def test_take_bytes_reentrant_resize(self): + # gh-153570: n.__index__() can resize the bytearray, so take_bytes() + # must re-read the size afterwards. It cached the size before the + # call and used it for the bounds check and the buffer reads, so a + # reentrant clear() returned freed memory (a use-after-free read). + def take(target, resize, n): + class Evil: + def __index__(self): + resize(target) + return n + return target.take_bytes(Evil()) + + # clear() during __index__: nothing is left to take. + ba = bytearray(b'abcdefgh') + with self.assertRaises(IndexError): + take(ba, lambda b: b.clear(), 8) + self.assertEqual(ba, b'') + + # shrink during __index__: n past the new size is out of range. + ba = bytearray(b'abcdefgh') + with self.assertRaises(IndexError): + take(ba, lambda b: b.__delitem__(slice(4, None)), 8) + self.assertEqual(ba, b'abcd') + + # grow during __index__: the take runs against the new, larger size. + ba = bytearray(b'abcd') + self.assertEqual(take(ba, lambda b: b.extend(b'efgh'), 8), b'abcdefgh') + self.assertEqual(ba, b'') + def test_setitem(self): def setitem_as_mapping(b, i, val): b[i] = val @@ -2197,6 +2226,46 @@ def __len__(self): self.assertRaises(BufferError, ba.hex, S(b':')) + def test_no_init_called(self): + # A bytearray created without calling bytearray.__init__ + # should not crash the interpreter (see gh-153419). + def bytearray_new(): + return bytearray.__new__(bytearray) + + bytearray_new().insert(0, 1) + bytearray_new().extend(b"x") + bytearray_new().extend([1, 2, 3]) + bytearray_new().resize(4) + bytearray_new().__init__(5) + bytearray_new().__init__(b"xyz") + bytearray_new().take_bytes() + bytearray_new().take_bytes(0) + + a = bytearray_new() + a.append(1) + + a = bytearray_new() + a += b"x" + + a = bytearray_new() + a[:] = b"xyz" + + def test_reinit_length(self): + # There is a shortcut taken when resizing, where alloc/2 < newsize. + # In this case, the existing buffer is reused, rather than reset. + # If this happens when newsize == 0 and alloc == 1, then various + # code assumptions can be violated. This test should catch those + # in debug builds. (see gh-153419) + a = bytearray(1) + a.__init__() + self.assertEqual(a, b"") + + def test_reinit_with_view(self): + a = bytearray() + with memoryview(a): + self.assertRaises(BufferError, a.__init__, "x", "ascii") + self.assertEqual(a, b"") + class AssortedBytesTest(unittest.TestCase): # diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index f42526aee194174..1e42356be21ddd6 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -3,7 +3,7 @@ set_recursion_limit, skip_on_s390x, skip_emscripten_stack_overflow, skip_wasi_stack_overflow, skip_if_sanitizer, - import_helper) + skip_if_huge_c_stack, import_helper) try: import _testcapi except ImportError: @@ -1062,6 +1062,7 @@ def get_sp(): @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") @skip_if_sanitizer("requires deep stack", thread=True) @unittest.skipIf(_testcapi is None, "requires _testcapi") + @skip_if_huge_c_stack(90_000) @skip_emscripten_stack_overflow() @skip_wasi_stack_overflow() def test_super_deep(self): diff --git a/Lib/test/test_capi/test_capsule.py b/Lib/test/test_capi/test_capsule.py new file mode 100644 index 000000000000000..981caf3fad426bd --- /dev/null +++ b/Lib/test/test_capi/test_capsule.py @@ -0,0 +1,181 @@ +import importlib +import os +import sys +import textwrap +import unittest +from test.support import import_helper, os_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + + +class CapsuleImportTests(unittest.TestCase): + """Tests for PyCapsule_Import().""" + + @classmethod + def setUpClass(cls): + tmp = cls.tmp = cls.enterClassContext(os_helper.temp_dir()) + cls.enterClassContext(import_helper.DirsOnSysPath(tmp)) + cls.write_file(os.path.join(tmp, 'capsule_mod.py'), ''' + import _testlimitedcapi + + capsule = _testlimitedcapi.capsule_new('capsule_mod.capsule') + капсула = _testlimitedcapi.capsule_new('capsule_mod.капсула') + mismatched = _testlimitedcapi.capsule_new('other.name') + nonutf8 = _testlimitedcapi.capsule_new(b'capsule_mod.nonutf8\\xff') + nullname = _testlimitedcapi.capsule_new(None) + not_capsule = 42 + + class ns: + nested = _testlimitedcapi.capsule_new('capsule_mod.ns.nested') + + def __getattr__(name): + if name == 'bad_attr': + raise FloatingPointError('bad attribute') + raise AttributeError(name) + ''') + pkg = os.path.join(tmp, 'capsule_pkg') + os.mkdir(pkg) + cls.write_file(os.path.join(pkg, '__init__.py'), '') + cls.write_file(os.path.join(pkg, 'sub.py'), ''' + import _testlimitedcapi + + capsule = _testlimitedcapi.capsule_new('capsule_pkg.sub.capsule') + ''') + autopkg = os.path.join(tmp, 'capsule_autopkg') + os.mkdir(autopkg) + cls.write_file(os.path.join(autopkg, '__init__.py'), 'from . import sub\n') + cls.write_file(os.path.join(autopkg, 'sub.py'), ''' + import _testlimitedcapi + + capsule = _testlimitedcapi.capsule_new('capsule_autopkg.sub.capsule') + ''') + cls.write_file(os.path.join(tmp, 'capsule_broken.py'), '1/0\n') + importlib.invalidate_caches() + + def setUp(self): + for name in ('capsule_mod', 'capsule_pkg.sub', 'capsule_pkg', + 'capsule_autopkg.sub', 'capsule_autopkg', + 'capsule_broken'): + self.addCleanup(import_helper.unload, name) + + @staticmethod + def write_file(path, source): + with open(path, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent(source)) + + def check_import(self, name, no_block=0): + # _testlimitedcapi.PyCapsule_Import() returns the name stored as the + # pointer by _testlimitedcapi.capsule_new(). + self.assertEqual(_testlimitedcapi.PyCapsule_Import(name, no_block), name) + + def test_import(self): + # The module is imported if not already imported. + self.assertNotIn('capsule_mod', sys.modules) + self.check_import('capsule_mod.capsule') + # Attributes after the first component are plain attribute lookups. + self.check_import('capsule_mod.ns.nested') + # Non-ASCII capsule and attribute name. + self.check_import('capsule_mod.капсула') + # The no_block argument is ignored. + self.check_import('capsule_mod.capsule', 1) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'requires non-ASCII file name support') + def test_non_ascii_module_name(self): + name = os_helper.TESTFN_NONASCII + self.write_file(os.path.join(self.tmp, name + '.py'), f''' + import _testlimitedcapi + + capsule = _testlimitedcapi.capsule_new('{name}.capsule') + ''') + importlib.invalidate_caches() + self.addCleanup(import_helper.unload, name) + self.check_import(f'{name}.capsule') + + def test_submodule(self): + # Only the first component is imported; a submodule not imported + # by its package is not found. + self.assertRaises(AttributeError, + _testlimitedcapi.PyCapsule_Import, 'capsule_pkg.sub.capsule') + # It is found after explicit import. + importlib.import_module('capsule_pkg.sub') + self.check_import('capsule_pkg.sub.capsule') + # A submodule imported by its package is found. + self.check_import('capsule_autopkg.sub.capsule') + + def test_invalid_name(self): + pycapsule_import = _testlimitedcapi.PyCapsule_Import + # Non-existing module. + self.assertRaisesRegex(ImportError, + 'PyCapsule_Import could not import module "capsule_nonexistent"', + pycapsule_import, 'capsule_nonexistent.capsule') + # Non-UTF-8 module name. + self.assertRaisesRegex(ImportError, + 'PyCapsule_Import could not import module', + pycapsule_import, b'\xff\xfe.capsule') + # Empty module name. + self.assertRaisesRegex(ImportError, + 'PyCapsule_Import could not import module ""', + pycapsule_import, '.capsule_mod.capsule') + # Empty name. + self.assertRaisesRegex(ImportError, + 'PyCapsule_Import could not import module ""', + pycapsule_import, '') + # Only a dot. + self.assertRaisesRegex(ImportError, + 'PyCapsule_Import could not import module ""', + pycapsule_import, '.') + # Non-existing attribute. + self.assertRaises(AttributeError, + pycapsule_import, 'capsule_mod.nonexistent') + # Empty attribute name. + self.assertRaises(AttributeError, pycapsule_import, 'capsule_mod.') + # Consecutive dots. + self.assertRaises(AttributeError, + pycapsule_import, 'capsule_mod..capsule') + # Attribute of an object which is not a module. + self.assertRaises(AttributeError, + pycapsule_import, 'capsule_mod.not_capsule.capsule') + # No attribute name. + self.assertRaisesRegex(AttributeError, 'is not valid', + pycapsule_import, 'capsule_mod') + + # CRASHES pycapsule_import(NULL) + + def test_invalid_capsule(self): + pycapsule_import = _testlimitedcapi.PyCapsule_Import + # The attribute is not a capsule. + self.assertRaisesRegex(AttributeError, 'is not valid', + pycapsule_import, 'capsule_mod.not_capsule') + # The capsule name does not match the requested name. + self.assertRaisesRegex(AttributeError, 'is not valid', + pycapsule_import, 'capsule_mod.mismatched') + # The capsule name contains a byte not decodable from UTF-8. + self.assertRaisesRegex(AttributeError, 'is not valid', + pycapsule_import, 'capsule_mod.nonutf8') + # Even the exactly matching name fails: the attribute lookup + # requires a name decodable from UTF-8. + self.assertRaises(UnicodeDecodeError, + pycapsule_import, b'capsule_mod.nonutf8\xff') + # The capsule name is NULL. + self.assertRaisesRegex(AttributeError, 'is not valid', + pycapsule_import, 'capsule_mod.nullname') + + def test_error_from_import(self): + # The exception raised during importing the module is replaced + # with generic ImportError. + with self.assertRaises(ImportError) as cm: + _testlimitedcapi.PyCapsule_Import('capsule_broken.capsule') + self.assertEqual(str(cm.exception), + 'PyCapsule_Import could not import ' + 'module "capsule_broken"') + + def test_error_from_attribute_lookup(self): + self.assertRaises(FloatingPointError, + _testlimitedcapi.PyCapsule_Import, 'capsule_mod.bad_attr') + self.assertRaises(FloatingPointError, + _testlimitedcapi.PyCapsule_Import, 'capsule_mod.bad_attr.capsule') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 66ab072733e17bd..e6389b4e0c1e924 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -24,9 +24,12 @@ def check(self, code): out = assert_python_failure( '-c', code, PYTHONMALLOC=self.PYTHONMALLOC, - # FreeBSD: instruct jemalloc to not fill freed() memory - # with junk byte 0x5a, see JEMALLOC(3) + # Instruct the system allocator to not fill freed() memory + # with junk bytes: + # FreeBSD: jemalloc, see JEMALLOC(3). MALLOC_CONF="junk:false", + # OpenBSD: see MALLOC.CONF(5). + MALLOC_OPTIONS="j", ) stderr = out.err return stderr.decode('ascii', 'replace') @@ -102,7 +105,9 @@ def check_pyobject_is_freed(self, func_name): assert_python_ok( '-c', code, PYTHONMALLOC=self.PYTHONMALLOC, + # See the comment in check() above. MALLOC_CONF="junk:false", + MALLOC_OPTIONS="j", ) def test_pyobject_null_is_freed(self): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 6d84f0b8c305dfb..7d668843d07debc 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -3043,22 +3043,37 @@ def test_pack_version(self): def test_pack_full_version_ctypes(self): ctypes = import_helper.import_module('ctypes') - ctypes_func = ctypes.pythonapi.Py_PACK_FULL_VERSION - ctypes_func.restype = ctypes.c_uint32 - ctypes_func.argtypes = [ctypes.c_int] * 5 + import ctypes.util # noqa: F811 + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def Py_PACK_FULL_VERSION( + x: ctypes.c_int, + y: ctypes.c_int, + z: ctypes.c_int, + level: ctypes.c_int, + serial: ctypes.c_int, + ) -> ctypes.c_uint32: + pass + for *args, expected in self.full_cases: with self.subTest(hexversion=hex(expected)): - result = ctypes_func(*args) + result = Py_PACK_FULL_VERSION(*args) self.assertEqual(result, expected) def test_pack_version_ctypes(self): ctypes = import_helper.import_module('ctypes') - ctypes_func = ctypes.pythonapi.Py_PACK_VERSION - ctypes_func.restype = ctypes.c_uint32 - ctypes_func.argtypes = [ctypes.c_int] * 2 + import ctypes.util # noqa: F811 + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def Py_PACK_VERSION( + x: ctypes.c_int, + y: ctypes.c_int, + ) -> ctypes.c_uint32: + pass + for *args, expected in self.xy_cases: with self.subTest(hexversion=hex(expected)): - result = ctypes_func(*args) + result = Py_PACK_VERSION(*args) self.assertEqual(result, expected) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 25b2c393e6773de..5806216d46e7eb6 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -5127,6 +5127,16 @@ def f(): f" {executor} at offset {idx} rather" f" than expected _EXIT_TRACE") + def test_jit_shutdown_after_cold_executor_creation(self): + script_helper.assert_python_ok("-c", textwrap.dedent(f""" + def f(): + for x in range({TIER2_THRESHOLD + 3}): + for y in range({TIER2_THRESHOLD + 3}): + z = x + y + + f() + """), PYTHON_JIT="1") + def test_enter_executor_valid_op_arg(self): script_helper.assert_python_ok("-c", textwrap.dedent(""" import sys diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index e401941c6d69700..62d8806b75d9db0 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -555,6 +555,7 @@ class Custom: self.assertFalse(hasattr(o, "__call__")) self.assertFalse(hasattr(c, "__call__")) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def testSFBug532646(self): diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 7640e50f19b7839..25d6d1a248b4577 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -2,6 +2,7 @@ # Most tests are executed with environment variables ignored # See test_cmd_line_script.py for testing of script execution +import locale import os import re import subprocess @@ -369,9 +370,27 @@ def run_no_utf8_mode(arg): ) test_args = [valid_utf8, invalid_utf8] - for run_cmd in (run_default, run_c_locale, run_utf8_mode, - run_no_utf8_mode): - with self.subTest(run_cmd=run_cmd): + for run_cmd, encoding in ( + (run_default, sys.getfilesystemencoding()), + (run_c_locale, None), + (run_utf8_mode, None), + (run_no_utf8_mode, locale.getencoding()) + ): + with self.subTest(run_cmd=run_cmd.__name__): + # Arbitrary bytes round-trip through surrogateescape only in + # UTF-8 and single-byte encodings, not in a multibyte encoding + # such as EUC-JP. + if encoding is not None: + try: + lossless = len(bytes(range(256)).decode( + encoding, 'surrogateescape')) == 256 + except UnicodeError: + lossless = False + else: + lossless = True + if not lossless: + self.skipTest(f'{encoding} cannot losslessly ' + f'round-trip arbitrary bytes') for arg in test_args: proc = run_cmd(arg) self.assertEqual(proc.stdout.rstrip(), ascii(arg)) diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index bd2fa1d7b70421a..631e5fd2499b13e 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -1540,21 +1540,26 @@ async def afunc(): [(1,1,3)]) if check_impl_detail(cpython=True) and ctypes is not None: - py = ctypes.pythonapi - freefunc = ctypes.CFUNCTYPE(None,ctypes.c_voidp) - - RequestCodeExtraIndex = py.PyUnstable_Eval_RequestCodeExtraIndex - RequestCodeExtraIndex.argtypes = (freefunc,) - RequestCodeExtraIndex.restype = ctypes.c_ssize_t - - SetExtra = py.PyUnstable_Code_SetExtra - SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) - SetExtra.restype = ctypes.c_int - - GetExtra = py.PyUnstable_Code_GetExtra - GetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, - ctypes.POINTER(ctypes.c_voidp)) - GetExtra.restype = ctypes.c_int + import ctypes.util + freefunc = ctypes.CFUNCTYPE(None, ctypes.c_voidp) + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyUnstable_Eval_RequestCodeExtraIndex(free: freefunc) -> ctypes.c_ssize_t: + pass + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyUnstable_Code_SetExtra(code: ctypes.py_object, + index: ctypes.c_ssize_t, + extra: ctypes.c_voidp) -> ctypes.c_int: + pass + SetExtra = PyUnstable_Code_SetExtra + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyUnstable_Code_GetExtra(code: ctypes.py_object, + index: ctypes.c_ssize_t, + extra: ctypes.POINTER(ctypes.c_voidp)) -> ctypes.c_int: + pass + GetExtra = PyUnstable_Code_GetExtra LAST_FREED = None def myfree(ptr): @@ -1562,7 +1567,7 @@ def myfree(ptr): LAST_FREED = ptr FREE_FUNC = freefunc(myfree) - FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC) + FREE_INDEX = PyUnstable_Eval_RequestCodeExtraIndex(FREE_FUNC) # Make sure myfree sticks around at least as long as the interpreter, # since we (currently) can't unregister the function and leaving a # dangling pointer will cause a crash on deallocation of code objects if diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index c18b203f42f59f3..31704955df3e14e 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -36,21 +36,25 @@ def check(input, expect): self.assertEqual(coder(input), (expect, len(input))) return check -# On small versions of Windows like Windows IoT or Windows Nano Server not all codepages are present +# On small versions of Windows like Windows IoT or Windows Nano Server, +# not all codepages are present def is_code_page_present(cp): - from ctypes import POINTER, WINFUNCTYPE, WinDLL, Structure + from ctypes import POINTER, WINFUNCTYPE, WinDLL + from ctypes.util import struct from ctypes.wintypes import BOOL, BYTE, WCHAR, UINT, DWORD MAX_LEADBYTES = 12 # 5 ranges, 2 bytes ea., 0 term. MAX_DEFAULTCHAR = 2 # single or double byte MAX_PATH = 260 - class CPINFOEXW(Structure): - _fields_ = [("MaxCharSize", UINT), - ("DefaultChar", BYTE*MAX_DEFAULTCHAR), - ("LeadByte", BYTE*MAX_LEADBYTES), - ("UnicodeDefaultChar", WCHAR), - ("CodePage", UINT), - ("CodePageName", WCHAR*MAX_PATH)] + + @struct + class CPINFOEXW: + MaxCharSize: UINT + DefaultChar: BYTE * MAX_DEFAULTCHAR + LeadByte: BYTE * MAX_LEADBYTES + UnicodeDefaultChar: WCHAR + CodePage: UINT + CodePageName: WCHAR * MAX_PATH prototype = WINFUNCTYPE(BOOL, UINT, DWORD, POINTER(CPINFOEXW)) GetCPInfoEx = prototype(("GetCPInfoExW", WinDLL("kernel32"))) diff --git a/Lib/test/test_colorsys.py b/Lib/test/test_colorsys.py index 74d76294b0b4d46..9dfaef44a6a4508 100644 --- a/Lib/test/test_colorsys.py +++ b/Lib/test/test_colorsys.py @@ -42,6 +42,10 @@ def test_hsv_values(self): self.assertTripleEqual(hsv, colorsys.rgb_to_hsv(*rgb)) self.assertTripleEqual(rgb, colorsys.hsv_to_rgb(*hsv)) + # test 360 phase shift in hue + h, s, v = hsv + self.assertTripleEqual(rgb, colorsys.hsv_to_rgb(h + 1.0, s, v)) + def test_hls_roundtrip(self): for r in frange(0.0, 1.0, 0.2): for g in frange(0.0, 1.0, 0.2): @@ -89,6 +93,18 @@ def test_yiq_roundtrip(self): colorsys.yiq_to_rgb(*colorsys.rgb_to_yiq(*rgb)) ) + def test_yiq_to_rgb_clamping(self): + values = [ + # rgb, yiq (invalid YIQ values clamped to RGB range) + ((1.0, 0.0, 1.0), (0.0, 0.5, 1.0)), + ((0.0, 1.0, 0.0), (0.25, -1.0, -1.0)), + ((0.0, 0.0, 1.0), (0.0, -1.0, 0.5)) + ] + + for (rgb, yiq) in values: + with self.subTest(rgb=rgb, yiq=yiq): + self.assertTripleEqual(rgb, colorsys.yiq_to_rgb(*yiq)) + def test_yiq_values(self): values = [ # rgb, yiq diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index ecc69b5383e3f21..2c7b1181817cf55 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -724,6 +724,7 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_if_huge_c_stack(100_000 if sys.platform == "android" else 500_000) @support.skip_emscripten_stack_overflow() def test_compiler_recursion_limit(self): # Compiler frames are small diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index ca612db17ce8021..c9fedc7213256c7 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -1,5 +1,6 @@ import contextlib import logging +import multiprocessing import queue import time import unittest @@ -147,8 +148,8 @@ def test_spawn(self): self._test(ProcessPoolSpawnFailingInitializerTest) @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) - @unittest.skipIf(sys.platform == "cygwin", - "Forkserver is not available on Cygwin") + @unittest.skipUnless("forkserver" in multiprocessing.get_all_start_methods(), + "forkserver start method is not available") def test_forkserver(self): self._test(ProcessPoolForkserverFailingInitializerTest) diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index 006360c8d941c9d..9dc1d057e75b4bc 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -139,6 +139,8 @@ def get_context(self): self.skipTest("require unix system") if support.check_sanitizer(thread=True): self.skipTest("TSAN doesn't support threads after fork") + if "forkserver" not in multiprocessing.get_all_start_methods(): + self.skipTest("forkserver start method is not available") return super().get_context() def create_event(self): diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 98f56b5ae87f964..9455c9e00514ca4 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -377,6 +377,7 @@ def test_deepcopy_list(self): self.assertIsNot(x, y) self.assertIsNot(x[0], y[0]) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_list(self): @@ -406,6 +407,7 @@ def test_deepcopy_tuple_of_immutables(self): y = copy.deepcopy(x) self.assertIs(x, y) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_tuple(self): @@ -449,6 +451,7 @@ def test_deepcopy_frozendict(self): self.assertIsNot(x[0], y[0]) self.assertIs(y[0]['foo'], y) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_dict(self): @@ -609,6 +612,7 @@ def __eq__(self, other): self.assertIsNot(y, x) self.assertIsNot(y.foo, x.foo) + @support.skip_if_huge_c_stack() def test_deepcopy_reflexive_inst(self): class C: pass @@ -671,6 +675,7 @@ def __eq__(self, other): self.assertEqual(y, x) self.assertIsNot(y.foo, x.foo) + @support.skip_if_huge_c_stack() def test_reconstruct_reflexive(self): class C(object): pass diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 7327c1bd5f50530..91170cc16b3ac95 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -1415,6 +1415,9 @@ class TestSniffer(unittest.TestCase): sample18.append("v,twenty_one") # 'u' was not skipped sample18 = '\n'.join(sample18) + sample19 = ('time,title\r\n' + '2020-10-01,"Pocket - Save news, videos, stories and more"\r\n') + def test_issue43625(self): sniffer = csv.Sniffer() self.assertTrue(sniffer.has_header(self.sample12)) @@ -1443,13 +1446,16 @@ def test_has_header_checks_20_rows(self): def test_guess_quote_and_delimiter(self): sniffer = csv.Sniffer() - for header in (";'123;4';", "'123;4';", ";'123;4'", "'123;4'"): + for header in (";'123;4';", "'123;4';", ";'123;4'"): with self.subTest(header): dialect = sniffer.sniff(header, ",;") self.assertEqual(dialect.delimiter, ';') self.assertEqual(dialect.quotechar, "'") self.assertIs(dialect.doublequote, False) self.assertIs(dialect.skipinitialspace, False) + # A single quoted field is a single column without a delimiter. + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, "'123;4'", ",;") def test_sniff(self): sniffer = csv.Sniffer() @@ -1494,6 +1500,253 @@ def test_delimiters(self): sniffer.sniff, self.sample15) self.assertRaisesRegex(csv.Error, "Could not determine delimiter", sniffer.sniff, self.sample16) + dialect = sniffer.sniff(self.sample19) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.quotechar, '"') + + def test_sniff_escapechar(self): + # gh-83273: escaped delimiters make the delimiter frequencies + # inconsistent, but the escape character can be detected by trial + # parsing. + sniffer = csv.Sniffer() + sample = 'ab,cd\\,ef\ngh\\,ij,kl\nmn,op\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.escapechar, '\\') + self.assertIs(dialect.doublequote, False) + self.assertEqual(list(csv.reader(StringIO(sample), dialect)), + [['ab', 'cd,ef'], ['gh,ij', 'kl'], ['mn', 'op']]) + # No escape character in the sample -- none is detected. + dialect = sniffer.sniff('ab,cd\ngh,ij\n') + self.assertEqual(dialect.delimiter, ',') + self.assertIsNone(dialect.escapechar) + + def test_sniff_quoted_rows_among_unquoted(self): + # Rows which happen to consist of a single quoted or unquoted + # field must not be mistaken for a single column of quoted fields. + sniffer = csv.Sniffer() + sample = '"header line"\na|b\nc|d\ne|f\n' + self.assertEqual(sniffer.sniff(sample).delimiter, '|') + # Even if an unterminated quote breaks the quoted parse. + sample = '"header"\na|"b c\nd|e\nf|g\n' + self.assertEqual(sniffer.sniff(sample).delimiter, '|') + + def test_sniff_single_column_quoted(self): + # gh-98820: a sample consisting of a single column of quoted fields + # has no delimiter to detect, even if the quoted content contains + # characters which could pass for one. It also used to take + # quadratic time before failing. + sniffer = csv.Sniffer() + sample = '\n'.join('"%02d-%02d-%02d"' % (i, i, i) for i in range(50)) + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + # Even if none of the requested delimiters occurs in the sample. + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample, ',:|\t') + # A mix of quoted and unquoted single-field rows. + sample = '"abc"\ndef\n"ghi"\njkl\n' * 10 + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + # Even if the quoted fields contain delimiter characters. + sample = '"a,b"\ncd\n' * 20 + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + # Even if the content of the quoted fields parses consistently + # with the other quote character. + sample = '''"a-'b'-c"\n"d-'e'-f"\n''' * 10 + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, sample) + + def test_sniff_non_ascii_delimiter(self): + # gh-111820: an explicitly requested delimiter can be non-ASCII. + sniffer = csv.Sniffer() + sample = 'aaa\xacbbb\xacccc\nddd\xaceee\xacfff\nggg\xachhh\xaciii\n' + dialect = sniffer.sniff(sample, delimiters='\xac') + self.assertEqual(dialect.delimiter, '\xac') + + def test_sniff_preamble(self): + # A preamble (title or comment lines) before the data must not + # prevent detection, even if it is larger than the part of the + # sample which is parsed first. It is enough for the data rows + # to slightly outnumber the preamble lines. + sniffer = csv.Sniffer() + for n in 3, 24, 80, 320: + preamble = ''.join(f'Comment line {i:05}\n' for i in range(n)) + for ndata in n + 2, 2 * n + 5: + with self.subTest(preamble_lines=n, data_lines=ndata): + sample = preamble + 'aaa,bbb,ccc\n' * ndata + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + + def test_sniff_line_boundary_characters(self): + # Only '\r', '\n' and '\r\n' are row separators; characters like + # '\x1c' or '\x85', which str.splitlines() treats as line + # boundaries, are ordinary data for the reader -- '\x1c' can + # even be the delimiter. + sniffer = csv.Sniffer() + sample = 'aa\x1cbb\ncc\x1cdd\nee\x1cff\n' + self.assertEqual(sniffer.sniff(sample).delimiter, '\x1c') + sample = 'a\x85b,c\nd,e\nf,g\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(next(csv.reader(StringIO(sample), dialect)), + ['a\x85b', 'c']) + + def test_sniff_delimiter_in_quoted_field(self): + # gh-97611: a delimiter inside a quoted field should not win over + # the delimiter which actually separates the fields. + sniffer = csv.Sniffer() + sample = ( + 'Surname;First Name;Year of birth\n' + '"\tDoe;Jane"\t;1971\n' + ) + dialect = sniffer.sniff(sample, delimiters=',;\t') + self.assertEqual(dialect.delimiter, ';') + sample = ( + 'Surname;First Name;Year of birth\n' + '"\t";"\t";1971\n' + '"Le Trec";"Mary Ann";1486\n' + ) + dialect = sniffer.sniff(sample, delimiters=',;\t') + self.assertEqual(dialect.delimiter, ';') + self.assertEqual(dialect.quotechar, '"') + + def test_sniff_embedded_lists(self): + # gh-119123: commas in bracketed lists adjacent to quotes should + # not be mistaken for the delimiter. + sniffer = csv.Sniffer() + sample = ( + "id;is_sort;cost;group;merge\n" + "1;True;62.25;['345'];UNKNOWN\n" + "2;True;54.00;['235'];UNKNOWN\n" + "3;True;237.00;['567', '568'];UNKNOWN\n" + "4;True;46.50;['112', '112'];UNKNOWN\n" + ) + dialect = sniffer.sniff(sample, delimiters=',;') + self.assertEqual(dialect.delimiter, ';') + + def test_sniff_space_adjacent_to_quotes(self): + # gh-88843: a space adjacent to stray quotes should not be + # detected as the delimiter. + sniffer = csv.Sniffer() + sample = "a|b\nc| 'd\ne|' f" + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, '|') + + def test_sniff_crlf_lineterminator(self): + # gh-103925: a quote at the end of a \r\n-terminated line. + sniffer = csv.Sniffer() + sample = ( + 'Timestamp,URL,Title\r\n' + '2020-10-01 17:17:37+08:00,' + 'https://www.mozilla.org/en-US/firefox/welcome/2/,' + '"Pocket - Save news, videos, stories and more"\r\n' + ) + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.quotechar, '"') + + def test_sniff_excel_tab_with_quotes(self): + # gh-62029: tab-delimited data with a quoted field containing + # spaces and doubled quotes. + sniffer = csv.Sniffer() + sample = ('foo\tbar\t"baz ""quoted"" here"\tspam eggs\n' + 'ham\teggs\tx y\tz w\n') + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, '\t') + self.assertEqual(dialect.quotechar, '"') + self.assertIs(dialect.doublequote, True) + + def test_sniff_truncated_sample(self): + # A sample cut off in the middle of a quoted field should not + # spoil the detection. + sniffer = csv.Sniffer() + sample = '"a,a";"b"\n"c";"d"\n"e";"f,f\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ';') + self.assertEqual(dialect.quotechar, '"') + # Cut off in the middle of the last row. + sample = 'a,b,c\nd,e,f\ng,h,i\nj,k' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + # Cut off in the middle of an escaped sequence. + sample = 'ab,cd\\,ef\ngh\\,ij,kl\nmn,op\nqr,st\\' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.escapechar, '\\') + # Cut off at a line end in the middle of a quoted field. + sample = '"a","b"\n"c","d"\n"e","multi\nline' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') + # An unclosed quote consumes everything to the end of the sample, + # but the rows before it still count. + sample = 'a|b\nc|d\ne|f\ng|"h\ni|j\nk|l' + self.assertEqual(sniffer.sniff(sample).delimiter, '|') + + def test_sniff_skipinitialspace_quoted(self): + sniffer = csv.Sniffer() + sample = "'a': 'b': 'c'\n'd': 'e': 'f'\n" + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ':') + self.assertEqual(dialect.quotechar, "'") + self.assertIs(dialect.skipinitialspace, True) + + def test_sniff_skipinitialspace_quoted_fields(self): + # A quote is only a quote at the very start of a field, so not + # skipping the space splits the quoted field. + sniffer = csv.Sniffer() + sample = 'a, "b,c"\nd,e\nf,g\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.quotechar, '"') + self.assertIs(dialect.skipinitialspace, True) + self.assertEqual(next(csv.reader(StringIO(sample), dialect)), + ['a', 'b,c']) + + # But without a delimiter inside the quotes nothing is split, + # so only the padding counts. + sample = 'a, "b"\nd,e\nf,g\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.quotechar, '"') + self.assertIs(dialect.skipinitialspace, False) + self.assertEqual(next(csv.reader(StringIO(sample), dialect)), + ['a', ' "b"']) + + def test_sniff_skipinitialspace_data(self): + # The spaces are a part of the data if only some fields + # are padded. + sniffer = csv.Sniffer() + sample = 'a, b\nc,d\ne, f\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertIs(dialect.skipinitialspace, False) + self.assertEqual(next(csv.reader(StringIO(sample), dialect)), + ['a', ' b']) + + def test_sniff_skipinitialspace_not_skipped(self): + # A quoted or escaped space is not skipped, so it is not + # an evidence of the padding. + sniffer = csv.Sniffer() + sample = 'a," b"\nc, d\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.quotechar, '"') + self.assertIs(dialect.skipinitialspace, False) + self.assertEqual(list(csv.reader(StringIO(sample), dialect)), + [['a', ' b'], ['c', ' d']]) + + # The escaped delimiter forces the escapechar detection. + sample = 'a,\\ b\\,c\nd, e\n' + dialect = sniffer.sniff(sample) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.escapechar, '\\') + self.assertIs(dialect.skipinitialspace, False) + self.assertEqual(list(csv.reader(StringIO(sample), dialect)), + [['a', ' b,c'], ['d', ' e']]) + + def test_sniff_regex_backtracking(self): + # gh-109638: this artificial sample used to take minutes. + sniffer = csv.Sniffer() + sample = '"",' * 100 + '"' * 100 + '0' + '"' * 100 + '0' + self.assertEqual(sniffer.sniff(sample).delimiter, ',') def test_doublequote(self): sniffer = csv.Sniffer() @@ -1507,6 +1760,15 @@ def test_doublequote(self): self.assertFalse(dialect.doublequote) dialect = sniffer.sniff(self.sample9) self.assertTrue(dialect.doublequote) + # A doubled quote character in an unquoted field or an empty + # quoted field is not evidence of doubling. + dialect = sniffer.sniff('"x",a""b\n"y",c\n') + self.assertIs(dialect.doublequote, False) + dialect = sniffer.sniff('a,"",c\nd,"",f\n') + self.assertIs(dialect.doublequote, False) + # A doubled quote character inside a quoted field is. + dialect = sniffer.sniff('"a""b",c\n"d",e\n') + self.assertIs(dialect.doublequote, True) def test_guess_delimiter_crlf_not_chosen(self): # Ensure that we pick the real delimiter ("|") over "\r" in a tie. diff --git a/Lib/test/test_ctypes/struct_str_ann.py b/Lib/test/test_ctypes/struct_str_ann.py new file mode 100644 index 000000000000000..7a924825dd81eeb --- /dev/null +++ b/Lib/test/test_ctypes/struct_str_ann.py @@ -0,0 +1,15 @@ +from __future__ import annotations +from ctypes.util import struct +from ctypes import c_int + +class TestAnn: + x: c_int + +# Check that "from __future__ import annotations" works as expected +if not isinstance(TestAnn.__annotations__['x'], str): + raise Exception("annotations must be strings") + +@struct +class Point: + x: c_int + y: c_int diff --git a/Lib/test/test_ctypes/test_aligned_structures.py b/Lib/test/test_ctypes/test_aligned_structures.py index 50b4d729b9db8ad..d58d92a19eb33c9 100644 --- a/Lib/test/test_ctypes/test_aligned_structures.py +++ b/Lib/test/test_ctypes/test_aligned_structures.py @@ -3,29 +3,64 @@ BigEndianStructure, LittleEndianStructure, BigEndianUnion, LittleEndianUnion, Structure, ) +from ctypes.util import struct as struct_util import struct import unittest from ._support import StructCheckMixin +from test.support import subTests + + +def get_struct_base(endian): + if endian == 'big': + return BigEndianStructure + elif endian == 'little': + return LittleEndianStructure + elif endian == 'native': + return Structure + else: + raise ValueError('invalid endian') + +def get_union_base(endian): + if endian == 'big': + return BigEndianUnion + elif endian == 'little': + return LittleEndianUnion + else: + raise ValueError('invalid endian') + class TestAlignedStructures(unittest.TestCase, StructCheckMixin): - def test_aligned_string(self): - for base, e in ( - (LittleEndianStructure, "<"), - (BigEndianStructure, ">"), + @subTests("use_struct_util", [False, True]) + def test_aligned_string(self, use_struct_util): + for endian, e in ( + ('little', "<"), + ('big', ">"), ): data = bytearray(struct.pack(f"{e}i12x16s", 7, b"hello world!")) - class Aligned(base): - _align_ = 16 - _fields_ = [ - ('value', c_char * 12) - ] + if use_struct_util: + @struct_util(endian=endian, align=16) + class Aligned: + value: c_char * 12 + else: + base = get_struct_base(endian) + class Aligned(base): + _align_ = 16 + _fields_ = [ + ('value', c_char * 12) + ] self.check_struct(Aligned) - class Main(base): - _fields_ = [ - ('first', c_uint32), - ('string', Aligned), - ] + if use_struct_util: + @struct_util(endian=endian) + class Main: + first: c_uint32 + string: Aligned + else: + class Main(base): + _fields_ = [ + ('first', c_uint32), + ('string', Aligned), + ] self.check_struct(Main) main = Main.from_buffer(data) @@ -37,24 +72,39 @@ class Main(base): self.assertEqual(alignment(main.string), 16) self.assertEqual(alignment(main), 16) - def test_aligned_structures(self): - for base, data in ( - (LittleEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), - (BigEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + @subTests("use_struct_util", [False, True]) + def test_aligned_structures(self, use_struct_util): + for endian, data in ( + ('little', bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + ('big', bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), ): - class SomeBools(base): - _align_ = 4 - _fields_ = [ - ("bool1", c_ubyte), - ("bool2", c_ubyte), - ] + if use_struct_util: + @struct_util(endian=endian, align=4) + class SomeBools: + bool1: c_ubyte + bool2: c_ubyte + else: + base = get_struct_base(endian) + class SomeBools(base): + _align_ = 4 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ] self.check_struct(SomeBools) - class Main(base): - _fields_ = [ - ("x", c_ubyte), - ("y", SomeBools), - ("z", c_ubyte), - ] + if use_struct_util: + @struct_util(endian=endian) + class Main: + x: c_ubyte + y: SomeBools + z: c_ubyte + else: + class Main(base): + _fields_ = [ + ("x", c_ubyte), + ("y", SomeBools), + ("z", c_ubyte), + ] self.check_struct(Main) main = Main.from_buffer(data) @@ -69,57 +119,93 @@ class Main(base): self.assertEqual(Main.z.offset, 8) self.assertEqual(main.z, 7) - def test_negative_align(self): - for base in (Structure, LittleEndianStructure, BigEndianStructure): + @subTests("use_struct_util", [False, True]) + def test_negative_align(self, use_struct_util): + for endian in ('native', 'little', 'big'): with ( - self.subTest(base=base), + self.subTest(endian=endian), self.assertRaisesRegex( ValueError, '_align_ must be a non-negative integer', ) ): - class MyStructure(base): - _align_ = -1 - _fields_ = [] - - def test_zero_align_no_fields(self): - for base in (Structure, LittleEndianStructure, BigEndianStructure): - with self.subTest(base=base): - class MyStructure(base): - _align_ = 0 - _fields_ = [] + if use_struct_util: + @struct_util(endian=endian, align=-1) + class MyStructure: + pass + else: + base = get_struct_base(endian) + class MyStructure(base): + _align_ = -1 + _fields_ = [] + + @subTests("use_struct_util", [False, True]) + def test_zero_align_no_fields(self, use_struct_util): + for endian in ('native', 'little', 'big'): + with self.subTest(endian=endian): + if use_struct_util: + @struct_util(endian=endian, align=0) + class MyStructure: + pass + else: + base = get_struct_base(endian) + class MyStructure(base): + _align_ = 0 + _fields_ = [] self.assertEqual(alignment(MyStructure), 1) self.assertEqual(alignment(MyStructure()), 1) - def test_zero_align_with_fields(self): - for base in (Structure, LittleEndianStructure, BigEndianStructure): - with self.subTest(base=base): - class MyStructure(base): - _align_ = 0 - _fields_ = [ - ("x", c_ubyte), - ] + @subTests("use_struct_util", [False, True]) + def test_zero_align_with_fields(self, use_struct_util): + for endian in ('native', 'little', 'big'): + with self.subTest(endian=endian): + if use_struct_util: + @struct_util(endian=endian, align=0) + class MyStructure: + x: c_ubyte + else: + base = get_struct_base(endian) + class MyStructure(base): + _align_ = 0 + _fields_ = [ + ("x", c_ubyte), + ] self.assertEqual(alignment(MyStructure), 1) self.assertEqual(alignment(MyStructure()), 1) - def test_oversized_structure(self): + @subTests("use_struct_util", [False, True]) + def test_oversized_structure(self, use_struct_util): data = bytearray(b"\0" * 8) - for base in (LittleEndianStructure, BigEndianStructure): - class SomeBoolsTooBig(base): - _align_ = 8 - _fields_ = [ - ("bool1", c_ubyte), - ("bool2", c_ubyte), - ("bool3", c_ubyte), - ] + for endian in ('little', 'big'): + if use_struct_util: + @struct_util(endian=endian, align=8) + class SomeBoolsTooBig: + bool1: c_ubyte + bool2: c_ubyte + bool3: c_ubyte + else: + base = get_struct_base(endian) + class SomeBoolsTooBig(base): + _align_ = 8 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ("bool3", c_ubyte), + ] self.check_struct(SomeBoolsTooBig) - class Main(base): - _fields_ = [ - ("y", SomeBoolsTooBig), - ("z", c_uint32), - ] + if use_struct_util: + @struct_util(endian=endian) + class Main: + y: SomeBoolsTooBig + z: c_uint32 + else: + class Main(base): + _fields_ = [ + ("y", SomeBoolsTooBig), + ("z", c_uint32), + ] self.check_struct(Main) with self.assertRaises(ValueError) as ctx: Main.from_buffer(data) @@ -128,31 +214,49 @@ class Main(base): 'Buffer size too small (4 instead of at least 8 bytes)' ) - def test_aligned_subclasses(self): - for base, e in ( - (LittleEndianStructure, "<"), - (BigEndianStructure, ">"), + @subTests("use_struct_util", [False, True]) + def test_aligned_subclasses(self, use_struct_util): + for endian, e in ( + ('little', "<"), + ('big', ">"), ): data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) - class UnalignedSub(base): - x: c_uint32 - _fields_ = [ - ("x", c_uint32), - ] + if use_struct_util: + @struct_util(endian=endian) + class UnalignedSub: + x: c_uint32 + else: + base = get_struct_base(endian) + class UnalignedSub(base): + x: c_uint32 + _fields_ = [ + ("x", c_uint32), + ] self.check_struct(UnalignedSub) - class AlignedStruct(UnalignedSub): - _align_ = 8 - _fields_ = [ - ("y", c_uint32), - ] + if use_struct_util: + @struct_util(endian=endian, align=8) + class AlignedStruct(UnalignedSub): + y: c_uint32 + else: + class AlignedStruct(UnalignedSub): + _align_ = 8 + _fields_ = [ + ("y", c_uint32), + ] self.check_struct(AlignedStruct) - class Main(base): - _fields_ = [ - ("a", c_uint32), - ("b", AlignedStruct) - ] + if use_struct_util: + @struct_util(endian=endian) + class Main: + a: c_uint32 + b: AlignedStruct + else: + class Main(base): + _fields_ = [ + ("a", c_uint32), + ("b", AlignedStruct) + ] self.check_struct(Main) main = Main.from_buffer(data) @@ -166,12 +270,14 @@ class Main(base): self.assertEqual(Main.b.offset, 8) self.assertEqual(Main.b.size, 8) - def test_aligned_union(self): - for sbase, ubase, e in ( - (LittleEndianStructure, LittleEndianUnion, "<"), - (BigEndianStructure, BigEndianUnion, ">"), + @subTests("use_struct_util", [False, True]) + def test_aligned_union(self, use_struct_util): + for sendian, uendian, e in ( + ('little', 'little', "<"), + ('big', 'big', ">"), ): data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + ubase = get_union_base(uendian) class AlignedUnion(ubase): _align_ = 8 _fields_ = [ @@ -180,11 +286,18 @@ class AlignedUnion(ubase): ] self.check_union(AlignedUnion) - class Main(sbase): - _fields_ = [ - ("first", c_uint32), - ("union", AlignedUnion), - ] + if use_struct_util: + @struct_util(endian=sendian) + class Main: + first: c_uint32 + union: AlignedUnion + else: + sbase = get_struct_base(sendian) + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", AlignedUnion), + ] self.check_struct(Main) main = Main.from_buffer(data) @@ -196,20 +309,29 @@ class Main(sbase): self.assertEqual(alignment(main.union), 8) self.assertEqual(alignment(main), 8) - def test_aligned_struct_in_union(self): - for sbase, ubase, e in ( - (LittleEndianStructure, LittleEndianUnion, "<"), - (BigEndianStructure, BigEndianUnion, ">"), + @subTests("use_struct_util", [False, True]) + def test_aligned_struct_in_union(self, use_struct_util): + for sendian, uendian, e in ( + ('little', 'little', "<"), + ('big', 'big', ">"), ): data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) - class Sub(sbase): - _align_ = 8 - _fields_ = [ - ("x", c_uint32), - ("y", c_uint32), - ] + if use_struct_util: + @struct_util(endian=sendian, align=8) + class Sub: + x: c_uint32 + y: c_uint32 + else: + sbase = get_struct_base(sendian) + class Sub(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint32), + ("y", c_uint32), + ] self.check_struct(Sub) + ubase = get_union_base(uendian) class MainUnion(ubase): _fields_ = [ ("a", c_uint32), @@ -217,11 +339,17 @@ class MainUnion(ubase): ] self.check_union(MainUnion) - class Main(sbase): - _fields_ = [ - ("first", c_uint32), - ("union", MainUnion), - ] + if use_struct_util: + @struct_util(endian=sendian) + class Main: + first: c_uint32 + union: MainUnion + else: + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", MainUnion), + ] self.check_struct(Main) main = Main.from_buffer(data) @@ -235,12 +363,14 @@ class Main(sbase): self.assertEqual(main.union.b.x, 3) self.assertEqual(main.union.b.y, 4) - def test_smaller_aligned_subclassed_union(self): - for sbase, ubase, e in ( - (LittleEndianStructure, LittleEndianUnion, "<"), - (BigEndianStructure, BigEndianUnion, ">"), + @subTests("use_struct_util", [False, True]) + def test_smaller_aligned_subclassed_union(self, use_struct_util): + for sendian, uendian, e in ( + ('little', 'little', "<"), + ('big', 'big', ">"), ): data = bytearray(struct.pack(f"{e}H2xI", 1, 0xD60102D7)) + ubase = get_union_base(uendian) class SubUnion(ubase): _align_ = 2 _fields_ = [ @@ -255,11 +385,18 @@ class MainUnion(SubUnion): ] self.check_union(SubUnion) - class Main(sbase): - _fields_ = [ - ("first", c_uint16), - ("union", MainUnion), - ] + if use_struct_util: + @struct_util(endian=sendian) + class Main: + first: c_uint16 + union: MainUnion + else: + sbase = get_struct_base(sendian) + class Main(sbase): + _fields_ = [ + ("first", c_uint16), + ("union", MainUnion), + ] self.check_struct(Main) main = Main.from_buffer(data) @@ -273,11 +410,12 @@ class Main(sbase): self.assertEqual(Main.first.size, 2) def test_larger_aligned_subclassed_union(self): - for ubase, e in ( - (LittleEndianUnion, "<"), - (BigEndianUnion, ">"), + for uendian, e in ( + ('little', "<"), + ('big', ">"), ): data = bytearray(struct.pack(f"{e}I4x", 0xD60102D6)) + ubase = get_union_base(uendian) class SubUnion(ubase): _align_ = 8 _fields_ = [ @@ -299,29 +437,44 @@ class Main(SubUnion): self.assertEqual(main.unsigned, 0xD6) self.assertEqual(main.signed, -42) - def test_aligned_packed_structures(self): - for sbase, e in ( - (LittleEndianStructure, "<"), - (BigEndianStructure, ">"), + @subTests("use_struct_util", [False, True]) + def test_aligned_packed_structures(self, use_struct_util): + for sendian, e in ( + ('little', "<"), + ('big', ">"), ): data = bytearray(struct.pack(f"{e}B2H4xB", 1, 2, 3, 4)) - class Inner(sbase): - _align_ = 8 - _fields_ = [ - ("x", c_uint16), - ("y", c_uint16), - ] + if use_struct_util: + @struct_util(endian=sendian, align=8) + class Inner: + x: c_uint16 + y: c_uint16 + else: + sbase = get_struct_base(sendian) + class Inner(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint16), + ("y", c_uint16), + ] self.check_struct(Inner) - class Main(sbase): - _pack_ = 1 - _layout_ = "ms" - _fields_ = [ - ("a", c_ubyte), - ("b", Inner), - ("c", c_ubyte), - ] + if use_struct_util: + @struct_util(endian=sendian, pack=1, layout="ms") + class Main: + a: c_ubyte + b: Inner + c: c_ubyte + else: + class Main(sbase): + _pack_ = 1 + _layout_ = "ms" + _fields_ = [ + ("a", c_ubyte), + ("b", Inner), + ("c", c_ubyte), + ] self.check_struct(Main) main = Main.from_buffer(data) diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index c9d728e9ae2f9cb..abbfb6efdcea55b 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -5,7 +5,8 @@ c_short, c_int, c_long, c_longlong, c_byte, c_wchar, c_float, c_double, ArgumentError) -from test.support import import_helper, skip_if_sanitizer, skip_emscripten_stack_overflow +from test.support import (import_helper, skip_if_sanitizer, + skip_emscripten_stack_overflow, skip_if_huge_c_stack) _ctypes_test = import_helper.import_module("_ctypes_test") @@ -193,6 +194,7 @@ class S8I(Structure): (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) @skip_if_sanitizer('requires deep stack', thread=True) + @skip_if_huge_c_stack() @skip_emscripten_stack_overflow() def test_recursive_as_param(self): class A: diff --git a/Lib/test/test_ctypes/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py index be641da30eadae1..28ff34f9048454d 100644 --- a/Lib/test/test_ctypes/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -2,6 +2,7 @@ import unittest from ctypes import (CDLL, Structure, CFUNCTYPE, sizeof, _CFuncPtr, c_void_p, c_char_p, c_char, c_int, c_uint, c_long) +from ctypes.util import wrap_dll_function from test.support import import_helper _ctypes_test = import_helper.import_module("_ctypes_test") from ._support import (_CData, PyCFuncPtrType, Py_TPFLAGS_DISALLOW_INSTANTIATION, @@ -130,6 +131,33 @@ def c_string(init): def test_abstract(self): self.assertRaises(TypeError, _CFuncPtr, 13, "name", 42, "iid") + def test_wrap_dll_function(self): + @wrap_dll_function(ctypes.pythonapi) + def PyObject_GetAttr(op: ctypes.py_object, attr: ctypes.py_object) -> ctypes.py_object: + """Call the PythonAPI function underlying getattr""" + pass + + class Foo: + a = "abc" + + self.assertEqual(PyObject_GetAttr(Foo, "a"), "abc") + self.assertEqual(PyObject_GetAttr.__doc__, "Call the PythonAPI function underlying getattr") + + with self.assertRaises(AttributeError): + @wrap_dll_function(ctypes.pythonapi) + def noexist(): + pass + + with self.assertRaisesRegex(ValueError, "'PyObject_GetAttrString' missing return type annotation"): + @wrap_dll_function(ctypes.pythonapi) + def PyObject_GetAttrString(op: ctypes.py_object, attr: ctypes.c_char_p): + pass + + def test_wrap_dll_function_str_ann(self): + from test.test_ctypes import wrap_str_ann + version = wrap_str_ann.Py_GetVersion() + self.assertIsInstance(version, bytes) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index 771cc8fbe0ec934..78480af34474a54 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -11,6 +11,7 @@ c_long, c_ulong, c_longlong, c_ulonglong, c_float, c_double) from ctypes import _pointer_type_cache, _pointer_type_cache_fallback +from test import support from test.support import import_helper from weakref import WeakSet _ctypes_test = import_helper.import_module("_ctypes_test") @@ -409,10 +410,9 @@ class BadType(ctypes._Pointer): pass func = ctypes.pythonapi.Py_GetVersion - func.argtypes = (BadType,) - - with self.assertRaises(ctypes.ArgumentError): - func(object()) + with support.swap_attr(func, 'argtypes', (BadType,)): + with self.assertRaises(ctypes.ArgumentError): + func(object()) class PointerTypeCacheTestCase(unittest.TestCase): # dummy tests to check warnings and base behavior diff --git a/Lib/test/test_ctypes/test_refcounts.py b/Lib/test/test_ctypes/test_refcounts.py index 1815649ceb5fff9..5191862658c5671 100644 --- a/Lib/test/test_ctypes/test_refcounts.py +++ b/Lib/test/test_ctypes/test_refcounts.py @@ -54,8 +54,10 @@ def func(*args): gc.collect() self.assertEqual(sys.getrefcount(func), orig_refcount) - class X(ctypes.Structure): - _fields_ = [("a", OtherCallback)] + @ctypes.util.struct + class X: + a: OtherCallback + x = X() x.a = OtherCallback(func) @@ -127,9 +129,9 @@ class PyObjectRestypeTest(unittest.TestCase): def test_restype_py_object_with_null_return(self): # Test that a function which returns a NULL PyObject * # without setting an exception does not crash. - PyErr_Occurred = ctypes.pythonapi.PyErr_Occurred - PyErr_Occurred.argtypes = [] - PyErr_Occurred.restype = ctypes.py_object + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyErr_Occurred() -> ctypes.py_object: + pass # At this point, there's no exception set, so PyErr_Occurred # returns NULL. Given the restype is py_object, the diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 4dc1ea867c0700a..a74a3548ee78413 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -88,13 +88,14 @@ class X(Structure): self.assertEqual(sizeof(X), min(8, longlong_align) + longlong_size) self.assertEqual(X.b.offset, min(8, longlong_align)) - with self.assertRaises(ValueError): - if use_struct_util: + if use_struct_util: + with self.assertRaises(NameError): @struct_util(pack=-1, layout='ms') class X: a: "b" b: "q" - else: + else: + with self.assertRaises(ValueError): class X(Structure): _fields_ = [("a", "b"), ("b", "q")] _pack_ = -1 @@ -954,6 +955,12 @@ class Foo: self.assertEqual(Foo.__name__, "Foo") + def test_string_annotations(self): + from test.test_ctypes import struct_str_ann + Point = struct_str_ann.Point + fields = [('x', c_int), ('y', c_int)] + self.assertEqual(Point._fields_, fields) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ctypes/test_unicode.py b/Lib/test/test_ctypes/test_unicode.py index d9e17371d135724..8e91f8f2c566803 100644 --- a/Lib/test/test_ctypes/test_unicode.py +++ b/Lib/test/test_ctypes/test_unicode.py @@ -1,4 +1,4 @@ -import ctypes +import ctypes.util import unittest from test.support import import_helper _ctypes_test = import_helper.import_module("_ctypes_test") @@ -26,8 +26,10 @@ def test_buffers(self): self.assertEqual(buf[6:5:-1], "") def test_embedded_null(self): - class TestStruct(ctypes.Structure): - _fields_ = [("unicode", ctypes.c_wchar_p)] + @ctypes.util.struct + class TestStruct: + unicode: ctypes.c_wchar_p + t = TestStruct() # This would raise a ValueError: t.unicode = "foo\0bar\0\0" diff --git a/Lib/test/test_ctypes/test_win32_com_foreign_func.py b/Lib/test/test_ctypes/test_win32_com_foreign_func.py index 7e54f8f6c31d33f..c797ab98cf1e424 100644 --- a/Lib/test/test_ctypes/test_win32_com_foreign_func.py +++ b/Lib/test/test_ctypes/test_win32_com_foreign_func.py @@ -1,4 +1,4 @@ -import ctypes +import ctypes.util import gc import sys import unittest @@ -20,14 +20,13 @@ E_NOINTERFACE = -2147467262 -class GUID(ctypes.Structure): +@ctypes.util.struct +class GUID: # https://learn.microsoft.com/en-us/windows/win32/api/guiddef/ns-guiddef-guid - _fields_ = [ - ("Data1", DWORD), - ("Data2", WORD), - ("Data3", WORD), - ("Data4", BYTE * 8), - ] + Data1: DWORD + Data2: WORD + Data3: WORD + Data4: BYTE * 8 def create_proto_com_method(name, index, restype, *argtypes): diff --git a/Lib/test/test_ctypes/wrap_str_ann.py b/Lib/test/test_ctypes/wrap_str_ann.py new file mode 100644 index 000000000000000..aa114a640c3a990 --- /dev/null +++ b/Lib/test/test_ctypes/wrap_str_ann.py @@ -0,0 +1,13 @@ +from __future__ import annotations +import ctypes.util + +def test_ann() -> ctypes.c_char_p: + ... + +# Check that "from __future__ import annotations" works as expected +if not isinstance(test_ann.__annotations__['return'], str): + raise Exception("annotations must be strings") + +@ctypes.util.wrap_dll_function(ctypes.pythonapi) +def Py_GetVersion() -> ctypes.c_char_p: + ... diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 3ba79be789cf3e5..9f7f8535b6d9a80 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -84,10 +84,15 @@ def wrapped(self, *args, **kwargs): term = os.environ.get('TERM') SHORT_MAX = 0x7fff -# ncurses before 6.5 can crash on repeated newterm(). Fall back to initscr() -# and skip the tests that need several screens. +# ncurses before 6.5, and the native curses of NetBSD and illumos/Solaris, +# crash on repeated newterm()/delscreen(); fall back to initscr() and skip the +# multi-screen tests. The native ones are keyed off the platform so a fixed +# version can be excluded later. _ncurses_version = getattr(curses, 'ncurses_version', None) -BROKEN_NEWTERM = _ncurses_version is not None and _ncurses_version < (6, 5) +if _ncurses_version is not None: + BROKEN_NEWTERM = _ncurses_version < (6, 5) +else: + BROKEN_NEWTERM = sys.platform.startswith(('netbsd', 'sunos')) USE_NEWTERM = hasattr(curses, 'newterm') and not BROKEN_NEWTERM # Older macOS reports a variation selector as a spacing character (wcwidth() @@ -785,13 +790,10 @@ def test_output_character(self): stdscr.move(2, 0) stdscr.echochar(v) self.assertEqual(self._read_char(2, 0), c) - # insch() round-trips a byte only where its code point equals - # the byte value (Latin-1): on a wide build ncurses winsch - # stores a printable byte directly as a code point instead of - # decoding it through the locale. - if ord(c) < 0x100: - stdscr.insch(1, 0, v) - self.assertEqual(self._read_char(1, 0), c) + # insch() decodes the byte through the locale like addch(), so + # it round-trips the same character. + stdscr.insch(1, 0, v) + self.assertEqual(self._read_char(1, 0), c) # The same characters supplied as a str. Unlike the int path above, a # str is stored as a wide-character cell on a wide build, so every @@ -958,10 +960,9 @@ def test_read_from_window(self): self.assertRaises(ValueError, stdscr.instr, -2) self.assertRaises(ValueError, stdscr.instr, 0, 2, -2) # A non-ASCII character of an 8-bit locale reads back as its encoded - # byte (see _encodable for the set). instr() returns the locale bytes - # for any single-byte character; inch() packs the text into a chtype, so - # on a wide build it only round-trips a Latin-1 codepoint (byte == - # codepoint). + # byte (see _encodable for the set). Both instr() and inch() return the + # locale byte for any character that fits the locale's single-byte + # encoding. encoding = stdscr.encoding for ch in ('A', 'é', '¤', '€', 'є'): try: @@ -973,8 +974,7 @@ def test_read_from_window(self): with self.subTest(ch=ch): stdscr.addstr(2, 0, ch) self.assertEqual(stdscr.instr(2, 0, 1), b) - if ord(ch) < 0x100: - self.assertEqual(stdscr.inch(2, 0) & curses.A_CHARTEXT, b[0]) + self.assertEqual(stdscr.inch(2, 0), b[0]) def test_coordinate_errors(self): # Addressing a cell outside the window raises curses.error. @@ -1145,8 +1145,17 @@ def test_attributes(self): win.standout() win.standend() + # attron()/attroff()/attrset() reject a bad attribute. + self.assertRaises(OverflowError, win.attron, 1 << 64) + self.assertRaises(OverflowError, win.attroff, -1) + self.assertRaises(OverflowError, win.attrset, 1 << 64) + self.assertRaises(TypeError, win.attron, 'x') + + @requires_curses_window_meth('attr_set') + def test_attr(self): # The attr_*() family works on attr_t attributes paired with a color # pair, unlike the chtype-based attron()/attroff()/attrset(). + win = curses.newwin(5, 15, 5, 2) win.attr_set(curses.A_BOLD | curses.A_UNDERLINE) attrs, pair = win.attr_get() self.assertTrue(attrs & curses.A_BOLD) @@ -1172,13 +1181,9 @@ def test_attributes(self): self.assertRaises(OverflowError, win.attr_set, -1) self.assertRaises(OverflowError, win.attr_on, -1) self.assertRaises(OverflowError, win.attr_set, 1 << 64) - # attron()/attroff()/attrset() reject a bad attribute too. - self.assertRaises(OverflowError, win.attron, 1 << 64) - self.assertRaises(OverflowError, win.attroff, -1) - self.assertRaises(OverflowError, win.attrset, 1 << 64) - self.assertRaises(TypeError, win.attron, 'x') @requires_colors + @requires_curses_window_meth('attr_set') def test_attr_color_pair(self): win = curses.newwin(5, 15, 5, 2) curses.init_pair(1, curses.COLOR_RED, curses.COLOR_BLACK) @@ -1388,8 +1393,10 @@ def test_scr_dump(self): stdscr.refresh() self.assertIsNone(curses.scr_restore(dump)) # scr_init() and scr_set() also accept a dump file and return None. + # scr_set() is not available on every curses (e.g. old SVr4). self.assertIsNone(curses.scr_init(dump)) - self.assertIsNone(curses.scr_set(dump)) + if hasattr(curses, 'scr_set'): + self.assertIsNone(curses.scr_set(dump)) # A bytes (path-like) filename is accepted too. curses.scr_dump(os.fsencode(dump)) # Restoring from a missing file is an error. @@ -1851,9 +1858,11 @@ def test_escdelay(self): def test_tabsize(self): tabsize = curses.get_tabsize() self.assertIsInstance(tabsize, int) - curses.set_tabsize(4) - self.assertEqual(curses.get_tabsize(), 4) - curses.set_tabsize(tabsize) + # set_tabsize() is not available on every curses (e.g. old SVr4). + if hasattr(curses, 'set_tabsize'): + curses.set_tabsize(4) + self.assertEqual(curses.get_tabsize(), 4) + curses.set_tabsize(tabsize) @requires_curses_func('getsyx') def test_getsyx(self): @@ -1988,12 +1997,15 @@ def test_dynamic_color_pairs(self): pair = curses.alloc_pair(fg, bg) self.assertGreater(pair, 0) self.assertEqual(curses.pair_content(pair), (fg, bg)) - # The same combination of colors reuses the same pair. - self.assertEqual(curses.alloc_pair(fg, bg), pair) - self.assertEqual(curses.find_pair(fg, bg), pair) - # Once freed, the pair is no longer found. - self.assertIsNone(curses.free_pair(pair)) - self.assertEqual(curses.find_pair(fg, bg), -1) + if getattr(curses, 'ncurses_version', (6, 3)) >= (6, 3): + # The same combination of colors reuses the same pair. + self.assertEqual(curses.alloc_pair(fg, bg), pair) + self.assertEqual(curses.find_pair(fg, bg), pair) + # Once freed, the pair is no longer found. + self.assertIsNone(curses.free_pair(pair)) + self.assertEqual(curses.find_pair(fg, bg), -1) + else: + self.assertIsNone(curses.free_pair(pair)) # Error paths. for color in self.bad_colors2(): @@ -2799,9 +2811,11 @@ def test_complexchar(self): # its single character. A narrow build just forms fewer cells. cc = curses.complexchar def storable(s): + # ValueError if s has combining marks on a narrow build. + # OverflowError if s is a multibyte character on a narrow build. try: cc(s) - except ValueError: + except (ValueError, OverflowError): return False return True @@ -3058,6 +3072,59 @@ def test_new_prescr(self): del screen gc_collect() + @requires_curses_func('new_prescr') + def test_set_term_prescr_screen(self): + # A new_prescr() screen has no terminal, so it cannot become the + # current one. It used to be accepted, and the next refresh then + # crashed inside curses. + s = self.make_pty() + screen = curses.newterm('xterm', s, s) + self.assertRaises(curses.error, curses.set_term, curses.new_prescr()) + # The current screen is unchanged, so refreshing it still works. + screen.stdscr.refresh() + + @unittest.skipUnless(hasattr(curses, 'new_prescr'), + 'requires curses.new_prescr()') + @unittest.skipUnless(hasattr(curses.screen, 'use'), + 'requires curses.screen.use()') + def test_use_prescr_screen(self): + # use() makes its screen current for the callback, so a new_prescr() + # screen is current there without having a terminal. Operations that + # need one used to crash inside curses. + s = self.make_pty() + screen = curses.newterm('xterm', s, s) + prescr = curses.new_prescr() + for func in [ + lambda scr: curses.doupdate(), + lambda scr: curses.newwin(3, 3), + lambda scr: screen.stdscr.refresh(), + lambda scr: screen.stdscr.getch(), + ]: + with self.assertRaises(curses.error): + prescr.use(func) + # Affecting the state before initscr() is what such a screen is for. + prescr.use(lambda scr: curses.use_env(False)) + # The current screen is unchanged. + screen.stdscr.refresh() + + def test_initscr_after_newterm_keeps_screen_alive(self): + # initscr() called while a newterm() screen is current returns that + # screen's own standard window, so the window keeps the screen alive. + # It used to be a second wrapper created without a screen: using it + # after the screen was collected read freed memory, and both wrappers + # could delwin() the same window. + s1 = self.make_pty() + s2 = self.make_pty() + screen1 = curses.newterm('xterm', s1, s1) + screen2 = curses.newterm('xterm', s2, s2) + curses.set_term(screen1) + win = curses.initscr() + self.assertIs(win, screen1.stdscr) + curses.set_term(screen2) + del screen1 + gc_collect() + win.addstr(0, 0, 'x') + @cpython_only def test_disallow_instantiation(self): # The screen type cannot be instantiated directly (bpo-43916). @@ -3119,7 +3186,10 @@ def test_set_wide(self): except UnicodeEncodeError: self.skipTest('the locale cannot encode %r' % label) curses.slk_set(1, label, 0) - self.assertEqual(curses.slk_label(1), label) + # The label can be truncated to fit the soft label width, e.g. in the + # EUC-JP locale, where "Å" and "ö" are double-width JIS X 0212 + # characters, so the 8-column label only fits "Ångstr". + self.assertIn(curses.slk_label(1), (label, 'Ångstr')) def test_set_bad_justify(self): self.make_slk_screen() diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index a04ed0c83c07c4a..1c723b25784da11 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4147,6 +4147,15 @@ def test_float_operation_default(self): @requires_cdecimal class CContextFlags(ContextFlags, unittest.TestCase): decimal = C + + def test_signaldict_repr(self): + Context = self.decimal.Context + ctx = Context(prec=7) + mapping = ctx.flags + del ctx + with self.assertRaisesRegex(ValueError, 'invalid signal dict'): + repr(mapping) + class PyContextFlags(ContextFlags, unittest.TestCase): decimal = P diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index ba504119d294fd3..7ac22455541fe2c 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -3774,6 +3774,7 @@ def f(a): return a encoding='latin1', errors='replace') self.assertEqual(ba, b'abc\xbd?') + @support.skip_if_huge_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() def test_recursive_call(self): @@ -5122,6 +5123,7 @@ class Thing: # CALL_METHOD_DESCRIPTOR_O deque.append(thing, thing) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_repr_as_str(self): diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 6d61c71e162f8be..dc31d403b837adb 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -678,6 +678,7 @@ def __repr__(self): d = {1: BadRepr()} self.assertRaises(Exc, repr, d) + @support.skip_if_huge_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() def test_repr_deep(self): diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 691fc5dfb5e8ab9..9816ae6c033ec74 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -2,7 +2,9 @@ import copy import pickle import unittest -from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, exceeds_recursion_limit +from test.support import (skip_emscripten_stack_overflow, + skip_wasi_stack_overflow, skip_if_huge_c_stack, + exceeds_recursion_limit) class DictSetTest(unittest.TestCase): @@ -277,6 +279,7 @@ def test_recursive_repr(self): # Again. self.assertIsInstance(r, str) + @skip_if_huge_c_stack() @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_deeply_nested_repr(self): diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 46c9b2c1d8c9fc4..4f99b7c91c654e4 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -284,6 +284,29 @@ def test_make_file_usascii_charset_with_nonascii_input(self): self.assertIn('charset="us-ascii"', output) self.assertIn('ımplıcıt', output) + def test_strip_trailing_newlines_before_diff(self): + # characterization test for the current buggy behavior + # see: gh-71896 + html_diff = difflib.HtmlDiff() + from_lines = [ + "Line 1: no newline after", + "Line 2: one newline after\n", + "Line 3: several newlines after\n\n\n\n\n", + ] + to_lines = [ + "Line 1: no newline after", + "Line 2: one newline after", # actually no \n + "Line 3: several newlines after", # actually no \n + ] + output = html_diff.make_table(from_lines, to_lines) + # we (currently) expect no line change, so all equal + self.assertNotIn('class="diff_add"', output) + self.assertNotIn('class="diff_chg"', output) + self.assertNotIn('class="diff_sub"', output) + self.assertEqual(output.count('>Line 1: no newline after<'), 2) + self.assertEqual(output.count('>Line 2: one newline after<'), 2) + self.assertEqual(output.count('>Line 3: several newlines after<'), 2) + class TestDiffer(unittest.TestCase): def test_close_matches_aligned(self): # Of the 4 closely matching pairs, we want 1 to match with 3, diff --git a/Lib/test/test_docxmlrpc.py b/Lib/test/test_docxmlrpc.py index 2ad422079b7f23e..64edc7b7e191d4c 100644 --- a/Lib/test/test_docxmlrpc.py +++ b/Lib/test/test_docxmlrpc.py @@ -127,7 +127,7 @@ def test_lambda(self): self.client.request("GET", "/") response = self.client.getresponse() - self.assertIn((b'
    ' + self.assertIn((b'
    ' b'<lambda>(x, y)
    '), response.read()) @@ -144,16 +144,17 @@ def test_autolinking(self): response = self.client.getresponse().read() self.assertIn( - (b'
    add(x, y)
    ' - b'Add two instances together. This ' - b'follows ' - b'PEP008, but has nothing
    \nto do ' - b'with ' - b'RFC1952. Case should matter: pEp008 ' - b'and rFC1952.  Things
    \nthat start ' - b'with http and ftp should be ' - b'auto-linked, too:
    \n' - b'http://google.com.
    '), response) + (b'
    add' + b'(x, y)
    ' + b'Add two instances together. This ' + b'follows ' + b'PEP008, but has nothing\nto do ' + b'with ' + b'RFC1952. Case should matter: pEp008 ' + b'and rFC1952. Things\nthat start ' + b'with http and ftp should be ' + b'auto-linked, too:\n' + b'http://google.com.
    '), response) @make_request_and_skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") @@ -167,22 +168,24 @@ def test_system_methods(self): response = self.client.getresponse().read() self.assertIn( - (b'
    system.methodHelp' - b'(method_name)
    system.methodHelp(\'add\') => "Adds ' - b'two integers together"
    \n 
    \nReturns a' - b' string containing documentation for ' - b'the specified method.
    \n
    system.methodSignature' - b'(method_name)
    ' - b'system.methodSignature(\'add\') => [double, ' - b'int, int]
    \n 
    \nReturns a list ' - b'describing the signature of the method.' - b' In the
    \nabove example, the add ' - b'method takes two integers as arguments' - b'
    \nand returns a double result.
    \n ' - b'
    \nThis server does NOT support system' - b'.methodSignature.
    '), response) + (b'
    ' + b'system.methodHelp(method_name)
    ' + b'
    system.methodHelp(\'add\') => "Adds ' + b'two integers together"\n\nReturns a' + b' string containing documentation for ' + b'the specified method.
    \n
    system.methodSignature' + b'(method_name)
    ' + b'' + b'system.methodSignature(\'add\') => [double, ' + b'int, int]\n\nReturns a list ' + b'describing the signature of the method.' + b' In the\nabove example, the add ' + b'method takes two integers as arguments' + b'\nand returns a double result.\n\n' + b'This server does NOT support system' + b'.methodSignature.
    '), response) def test_autolink_dotted_methods(self): """Test that selfdot values are made strong automatically in the @@ -190,7 +193,7 @@ def test_autolink_dotted_methods(self): self.client.request("GET", "/") response = self.client.getresponse() - self.assertIn(b"""Try self.add, too.""", + self.assertIn(b"""Try self.add, too.""", response.read()) def test_annotations(self): @@ -198,11 +201,11 @@ def test_annotations(self): self.client.request("GET", "/") response = self.client.getresponse() docstring = (b'' if sys.flags.optimize >= 2 else - b'
    Use function annotations.
    ') + b'
    Use function annotations.
    ') self.assertIn( - (b'
    annotation' - b'(x: int)
    ' + docstring + b'
    \n' - b'
    ' + (b'
    annotation' + b'(x: int)
    ' + docstring + b'
    \n' + b'
    ' b'method_annotation(x: bytes)
    '), response.read()) @@ -217,9 +220,11 @@ def test_server_title_escape(self): generated = self.serv.generate_html_documentation() title = re.search(r'(.+?)', generated).group() - documentation = re.search(r'

    (.+?)

    ', generated).group() + documentation = re.search(r'
    (.+?)
    ', + generated).group() self.assertEqual('Python: test_title<script>', title) - self.assertEqual('

    test_documentation<script>

    ', documentation) + self.assertEqual('
    test_documentation<script>
    ', + documentation) if __name__ == '__main__': diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 40036609a190551..1ff600e30bf4cbd 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1884,19 +1884,31 @@ def test_global_pathconfig(self): # The global path configuration (_Py_path_config) must be a copy # of the path configuration of PyInterpreter.config (PyConfig). ctypes = import_helper.import_module('ctypes') + import ctypes.util # noqa: F811 - def get_func(name): - func = getattr(ctypes.pythonapi, name) - func.argtypes = () - func.restype = ctypes.c_wchar_p - return func - - Py_GetPath = get_func('Py_GetPath') - Py_GetPrefix = get_func('Py_GetPrefix') - Py_GetExecPrefix = get_func('Py_GetExecPrefix') - Py_GetProgramName = get_func('Py_GetProgramName') - Py_GetProgramFullPath = get_func('Py_GetProgramFullPath') - Py_GetPythonHome = get_func('Py_GetPythonHome') + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def Py_GetPath() -> ctypes.c_wchar_p: + pass + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def Py_GetPrefix() -> ctypes.c_wchar_p: + pass + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def Py_GetExecPrefix() -> ctypes.c_wchar_p: + pass + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def Py_GetProgramName() -> ctypes.c_wchar_p: + pass + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def Py_GetProgramFullPath() -> ctypes.c_wchar_p: + pass + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def Py_GetPythonHome() -> ctypes.c_wchar_p: + pass config = _testinternalcapi.get_configs()['config'] diff --git a/Lib/test/test_enumerate.py b/Lib/test/test_enumerate.py index 5cb54cff9b76fdd..c8b85fe86921781 100644 --- a/Lib/test/test_enumerate.py +++ b/Lib/test/test_enumerate.py @@ -3,8 +3,11 @@ import sys import pickle import gc +import threading + from test import support +from test.support import threading_helper class G: 'Sequence using __getitem__' @@ -292,5 +295,28 @@ def enum(self, iterable, start=sys.maxsize + 1): (sys.maxsize+3,'c')] +@threading_helper.requires_working_threading() +class TestThreadSafety(EnumerateStartTestCase): + def test_thread_safety_while_iterating(self): + # gh-153932: calling reduce while iterating should pass with TSAN + + en = enumerate(range(10_000)) + stop = threading.Event() + + def advance(): + for _ in en: + pass + stop.set() + + def read(): + while not stop.is_set(): + en.__reduce__() + + threads = [threading.Thread(target=advance), threading.Thread(target=read)] + + with threading_helper.start_threads(threads): + pass + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_epoll.py b/Lib/test/test_epoll.py index 5e6a4ab0166a86f..51405ca7c9e57c3 100644 --- a/Lib/test/test_epoll.py +++ b/Lib/test/test_epoll.py @@ -25,6 +25,7 @@ import os import select import socket +import sys import time import unittest from test import support @@ -52,12 +53,11 @@ def tearDown(self): def _connected_pair(self): client = socket.socket() client.setblocking(False) + # The connection is either established immediately or in progress. try: client.connect(('127.0.0.1', self.serverSocket.getsockname()[1])) except OSError as e: self.assertEqual(e.args[0], errno.EINPROGRESS) - else: - raise AssertionError("Connect should have raised EINPROGRESS") server, addr = self.serverSocket.accept() self.connections.extend((client, server)) @@ -218,6 +218,9 @@ def test_errors(self): self.assertRaises(ValueError, select.epoll().register, -1, select.EPOLLIN) + @unittest.skipIf(sys.platform.startswith('sunos'), + 'unregistering a closed file descriptor succeeds ' + 'on Solaris') def test_unregister_closed(self): server, client = self._connected_pair() fd = server.fileno() diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 35ffc9a0a4cf30a..325b1c91fa5aeef 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,9 @@ import collections import types import unittest -from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, exceeds_recursion_limit +from test.support import (skip_emscripten_stack_overflow, + skip_wasi_stack_overflow, skip_if_huge_c_stack, + exceeds_recursion_limit) class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -547,6 +549,7 @@ def make_deep_eg(self): e = ExceptionGroup('eg', [e]) return e + @skip_if_huge_c_stack() @skip_emscripten_stack_overflow() @skip_wasi_stack_overflow() def test_deep_split(self): @@ -554,6 +557,7 @@ def test_deep_split(self): with self.assertRaises(RecursionError): e.split(TypeError) + @skip_if_huge_c_stack() @skip_emscripten_stack_overflow() @skip_wasi_stack_overflow() def test_deep_subgroup(self): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index cc7faef93e1af7c..0cee756958f3ad1 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -10,6 +10,7 @@ from codecs import BOM_UTF8 from itertools import product from textwrap import dedent +from types import ModuleType from test.support import (captured_stderr, check_impl_detail, cpython_only, gc_collect, @@ -440,10 +441,16 @@ def test_WindowsError(self): def test_windows_message(self): """Should fill in unknown error code in Windows error message""" ctypes = import_module('ctypes') + import ctypes.util # noqa: F811 + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyErr_SetFromWindowsErr(ierr: ctypes.c_int) -> ctypes.py_object: + pass + # this error code has no message, Python formats it as hexadecimal code = 3765269347 - with self.assertRaisesRegex(OSError, 'Windows Error 0x%x' % code): - ctypes.pythonapi.PyErr_SetFromWindowsErr(code) + with self.assertRaisesRegex(OSError, f'Windows Error 0x{code:x}'): + PyErr_SetFromWindowsErr(code) def testAttributes(self): # test that exception attributes are happy @@ -1517,6 +1524,7 @@ def gen(): @cpython_only @unittest.skipIf(_testcapi is None, "requires _testcapi") @force_not_colorized + @support.skip_if_huge_c_stack() def test_recursion_normalizing_infinite_exception(self): # Issue #30697. Test that a RecursionError is raised when # maximum recursion depth has been exceeded when creating @@ -2047,6 +2055,129 @@ def blech(self): self.assertEqual("bluch", exc.name) self.assertEqual(obj, exc.obj) + def test_getattr_error_message(self): + def fqn(type): + return f'{type.__module__}.{type.__qualname__}' + + class RaiseWithName: + def __getattr__(self, name): + raise AttributeError(name) + obj = RaiseWithName() + with self.assertRaises(AttributeError) as cm: + getattr(obj, "missing1") + self.assertEqual(str(cm.exception), + f"'{fqn(RaiseWithName)}' object has no attribute 'missing1'") + self.assertIs(cm.exception.obj, obj) + self.assertEqual(cm.exception.name, "missing1") + + class BareRaise: + def __getattr__(self, name): + raise AttributeError + obj = BareRaise() + with self.assertRaises(AttributeError) as cm: + getattr(obj, "missing2") + self.assertEqual(str(cm.exception), + f"'{fqn(BareRaise)}' object has no attribute 'missing2'") + self.assertIs(cm.exception.obj, obj) + self.assertEqual(cm.exception.name, "missing2") + + class RaiseCustom: + def __getattr__(self, name): + raise AttributeError("custom") + obj = RaiseCustom() + with self.assertRaises(AttributeError) as cm: + getattr(obj, "missing3") + self.assertEqual(str(cm.exception), "custom") + self.assertIs(cm.exception.obj, obj) + self.assertEqual(cm.exception.name, "missing3") + + def test_class_getattr_error_message(self): + def fqn(type): + return f'{type.__module__}.{type.__qualname__}' + + class MetaclassRaiseWithName(type): + def __getattr__(self, name): + raise AttributeError(name) + cls = MetaclassRaiseWithName("spam", (), {}) + with self.assertRaises(AttributeError) as cm: + getattr(cls, "missing1") + self.assertEqual(str(cm.exception), + f"type object '{fqn(cls)}' has no attribute 'missing1'") + self.assertIs(cm.exception.obj, cls) + self.assertEqual(cm.exception.name, "missing1") + + class MetaclassBareRaise(type): + def __getattr__(self, name): + raise AttributeError + cls = MetaclassBareRaise("eggs", (), {}) + with self.assertRaises(AttributeError) as cm: + getattr(cls, "missing2") + self.assertEqual(str(cm.exception), + f"type object '{fqn(cls)}' has no attribute 'missing2'") + self.assertIs(cm.exception.obj, cls) + self.assertEqual(cm.exception.name, "missing2") + + class MetaclassRaiseCustom(type): + def __getattr__(self, name): + raise AttributeError("custom") + cls = MetaclassRaiseCustom("ham", (), {}) + with self.assertRaises(AttributeError) as cm: + getattr(cls, "missing3") + self.assertEqual(str(cm.exception), "custom") + self.assertIs(cm.exception.obj, cls) + self.assertEqual(cm.exception.name, "missing3") + + def test_module_getattr_error_message(self): + raisewithname_mod = ModuleType("raisewithname") + def raise_with_name(name): + raise AttributeError(name) + raisewithname_mod.__getattr__ = raise_with_name + with self.assertRaises(AttributeError) as cm: + getattr(raisewithname_mod, "missing1") + self.assertEqual(str(cm.exception), + "module 'raisewithname' has no attribute 'missing1'") + self.assertIs(cm.exception.obj, raisewithname_mod) + self.assertEqual(cm.exception.name, "missing1") + + bareraise_mod = ModuleType("bareraise") + def bare_raise(name): + raise AttributeError + bareraise_mod.__getattr__ = bare_raise + with self.assertRaises(AttributeError) as cm: + getattr(bareraise_mod, "missing2") + self.assertEqual(str(cm.exception), + "module 'bareraise' has no attribute 'missing2'") + self.assertIs(cm.exception.obj, bareraise_mod) + self.assertEqual(cm.exception.name, "missing2") + + custom_mod = ModuleType("custom") + def raise_custom(name): + raise AttributeError("custom") + custom_mod.__getattr__ = raise_custom + with self.assertRaises(AttributeError) as cm: + getattr(custom_mod, "missing3") + self.assertEqual(str(cm.exception), "custom") + self.assertIs(cm.exception.obj, custom_mod) + self.assertEqual(cm.exception.name, "missing3") + + nameless_mod = ModuleType("forgettable") + del nameless_mod.__dict__["__name__"] + nameless_mod.__getattr__ = raise_with_name + with self.assertRaises(AttributeError) as cm: + getattr(nameless_mod, "missing4") + self.assertEqual(str(cm.exception), "module has no attribute 'missing4'") + self.assertIs(cm.exception.obj, nameless_mod) + self.assertEqual(cm.exception.name, "missing4") + + nameless_mod = ModuleType("broken") + nameless_mod.__dict__["__name__"] = 10j + nameless_mod.__getattr__ = raise_with_name + with self.assertRaises(AttributeError) as cm: + getattr(nameless_mod, "missing4") + self.assertEqual(str(cm.exception), "module has no attribute 'missing4'") + self.assertIs(cm.exception.obj, nameless_mod) + self.assertEqual(cm.exception.name, "missing4") + # Note: name suggestion tests live in `test_traceback`. diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 6b1529aa173f01c..910fe96d5e7d81e 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -438,6 +438,56 @@ def _extract_coroutine_stacks_lineno_only(self, stack_trace): # ============================================================================ +@requires_remote_subprocess_debugging() +class TestSelfStackTrace(RemoteInspectionTestBase): + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_self_trace_with_large_linetable(self): + script = textwrap.dedent("""\ + import os + import _remote_debugging + + assignments = "\\n".join( + f"value_{i} = {i}" for i in range(1000) + ) + expected_lineno = len(assignments.splitlines()) + 1 + source = ( + f"{assignments}\\n" + "stack_trace = " + "_remote_debugging.RemoteUnwinder(os.getpid()).get_stack_trace()\\n" + ) + code = compile(source, "large_linetable.py", "exec") + assert len(code.co_linetable) > 4096, len(code.co_linetable) + namespace = {"os": os, "_remote_debugging": _remote_debugging} + exec(code, namespace) + large_linetable_frames = [ + frame + for interpreter in namespace["stack_trace"] + for thread in interpreter.threads + for frame in thread.frame_info + if frame.filename == "large_linetable.py" + ] + assert len(large_linetable_frames) == 1, large_linetable_frames + assert large_linetable_frames[0].location.lineno == expected_lineno, ( + large_linetable_frames[0] + ) + """) + + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=SHORT_TIMEOUT, + ) + self.assertEqual( + result.returncode, 0, + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) + + @requires_remote_subprocess_debugging() class TestGetStackTrace(RemoteInspectionTestBase): @skip_if_not_supported diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index fc50e16575da2b9..a3a329cb578a8dc 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -821,28 +821,37 @@ class TestFrameCApi(unittest.TestCase): def test_basic(self): x = 1 ctypes = import_helper.import_module('ctypes') - PyEval_GetFrameLocals = ctypes.pythonapi.PyEval_GetFrameLocals - PyEval_GetFrameLocals.restype = ctypes.py_object + import ctypes.util # noqa: F811 + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyEval_GetFrameLocals() -> ctypes.py_object: + pass + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyEval_GetFrameGlobals() -> ctypes.py_object: + pass + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyEval_GetFrameBuiltins() -> ctypes.py_object: + pass + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyFrame_GetLocals(frame: ctypes.py_object) -> ctypes.py_object: + pass + frame_locals = PyEval_GetFrameLocals() self.assertTrue(type(frame_locals), dict) self.assertEqual(frame_locals['x'], 1) frame_locals['x'] = 2 self.assertEqual(x, 1) - PyEval_GetFrameGlobals = ctypes.pythonapi.PyEval_GetFrameGlobals - PyEval_GetFrameGlobals.restype = ctypes.py_object frame_globals = PyEval_GetFrameGlobals() self.assertTrue(type(frame_globals), dict) self.assertIs(frame_globals, globals()) - PyEval_GetFrameBuiltins = ctypes.pythonapi.PyEval_GetFrameBuiltins - PyEval_GetFrameBuiltins.restype = ctypes.py_object frame_builtins = PyEval_GetFrameBuiltins() self.assertEqual(frame_builtins, __builtins__) - PyFrame_GetLocals = ctypes.pythonapi.PyFrame_GetLocals - PyFrame_GetLocals.argtypes = [ctypes.py_object] - PyFrame_GetLocals.restype = ctypes.py_object frame = sys._getframe() f_locals = PyFrame_GetLocals(frame) self.assertTrue(f_locals['x'], 1) diff --git a/Lib/test/test_free_threading/test_capi.py b/Lib/test/test_free_threading/test_capi.py index 146d7cfc97adb7d..b0be94629af009d 100644 --- a/Lib/test/test_free_threading/test_capi.py +++ b/Lib/test/test_free_threading/test_capi.py @@ -1,4 +1,4 @@ -import ctypes +import ctypes.util import sys import unittest @@ -6,9 +6,9 @@ from test.support.threading_helper import run_concurrently -_PyImport_AddModuleRef = ctypes.pythonapi.PyImport_AddModuleRef -_PyImport_AddModuleRef.argtypes = (ctypes.c_char_p,) -_PyImport_AddModuleRef.restype = ctypes.py_object +@ctypes.util.wrap_dll_function(ctypes.pythonapi) +def PyImport_AddModuleRef(name: ctypes.c_char_p) -> ctypes.py_object: + pass @threading_helper.requires_working_threading() @@ -26,7 +26,7 @@ def test_pyimport_addmoduleref_thread_safe(self): results = [] def worker(): - module = _PyImport_AddModuleRef(module_name_bytes) + module = PyImport_AddModuleRef(module_name_bytes) results.append(module) for _ in range(NUM_ITERS): diff --git a/Lib/test/test_free_threading/test_code.py b/Lib/test/test_free_threading/test_code.py index 2fc5eea3773c399..121f00c1d7a92d9 100644 --- a/Lib/test/test_free_threading/test_code.py +++ b/Lib/test/test_free_threading/test_code.py @@ -12,25 +12,28 @@ from test.support.threading_helper import run_concurrently if ctypes is not None: - capi = ctypes.pythonapi + import ctypes.util freefunc = ctypes.CFUNCTYPE(None, ctypes.c_voidp) - RequestCodeExtraIndex = capi.PyUnstable_Eval_RequestCodeExtraIndex - RequestCodeExtraIndex.argtypes = (freefunc,) - RequestCodeExtraIndex.restype = ctypes.c_ssize_t - - SetExtra = capi.PyUnstable_Code_SetExtra - SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) - SetExtra.restype = ctypes.c_int - - GetExtra = capi.PyUnstable_Code_GetExtra - GetExtra.argtypes = ( - ctypes.py_object, - ctypes.c_ssize_t, - ctypes.POINTER(ctypes.c_voidp), - ) - GetExtra.restype = ctypes.c_int + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyUnstable_Eval_RequestCodeExtraIndex(free: freefunc) -> ctypes.c_ssize_t: + pass + RequestCodeExtraIndex = PyUnstable_Eval_RequestCodeExtraIndex + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyUnstable_Code_SetExtra(code: ctypes.py_object, + index: ctypes.c_ssize_t, + extra: ctypes.c_voidp) -> ctypes.c_int: + pass + SetExtra = PyUnstable_Code_SetExtra + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyUnstable_Code_GetExtra(code: ctypes.py_object, + index: ctypes.c_ssize_t, + extra: ctypes.POINTER(ctypes.c_voidp)) -> ctypes.c_int: + pass + GetExtra = PyUnstable_Code_GetExtra # Note: each call to RequestCodeExtraIndex permanently allocates a slot # (the counter is monotonically increasing), up to MAX_CO_EXTRA_USERS (255). diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index ad23290a92ab345..4a812275143bc4d 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -336,5 +336,25 @@ def reader(): with threading_helper.start_threads([t1, t2]): pass + def test_getstate_race_with_shared_keys(self): + box = [None] + enter = Barrier(2) + leave = Barrier(2) + + def reader(): + for _ in range(1000): + enter.wait() + box[0].__getstate__() + leave.wait() + + def writer(): + for i in range(1000): + box[0] = type(f"C{i}", (), {})() + enter.wait() + setattr(box[0], str(i), 1) + leave.wait() + + threading_helper.run_concurrently([reader, writer]) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_interpreters.py b/Lib/test/test_free_threading/test_interpreters.py new file mode 100644 index 000000000000000..a9a432fae16b1e1 --- /dev/null +++ b/Lib/test/test_free_threading/test_interpreters.py @@ -0,0 +1,29 @@ +import textwrap +import unittest + +from test.support import import_helper, script_helper + + +# Make sure _testinternalcapi is available before running the test. +import_helper.import_module('_testinternalcapi') + + +class InterpreterTeardownTests(unittest.TestCase): + def test_destroy_subinterpreter_does_not_abort(self): + # gh-153176: destroy_interpreter(basic=True) used to call + # PyThreadState_Clear() on a non-current thread state, which on a + # free-threaded debug build reclaimed mimalloc pages into a heap not + # owned by the current thread and aborted the process. Run the + # reproduction in a subprocess so that a regression surfaces as a + # non-zero exit / SIGABRT instead of killing the test runner. + script = textwrap.dedent(""" + import _testinternalcapi + + interpid = _testinternalcapi.create_interpreter() + _testinternalcapi.destroy_interpreter(interpid, basic=True) + """) + script_helper.assert_python_ok('-c', script) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_io.py b/Lib/test/test_free_threading/test_io.py index e0bd7e211e73024..057e0adf3b42bc4 100644 --- a/Lib/test/test_free_threading/test_io.py +++ b/Lib/test/test_free_threading/test_io.py @@ -122,6 +122,34 @@ def sizeof(barrier, b, *ignore): class CBytesIOTest(ThreadSafetyMixin, TestCase): ioclass = io.BytesIO + @threading_helper.requires_working_threading() + @threading_helper.reap_threads + def test_concurrent_setstate_and_method_call(self): + # gh-153290: __setstate__() installed the instance __dict__ with a + # plain store, racing the lock-free LOAD_ATTR method fast path that + # reads the dict slot with an atomic acquire load. + states = [(b"A" * 64, 0, {}), (b"B" * 128, 32, {}), (b"C" * 256, 0, {})] + nreaders = 4 + for _ in range(25): + shared = self.ioclass(b"initial payload") + barrier = threading.Barrier(1 + nreaders) + + def setter(): + barrier.wait() + for state in states: + shared.__setstate__(state) + + def reader(): + barrier.wait() + for _ in range(100): + shared.read(8) + + threads = [threading.Thread(target=setter)] + threads += [threading.Thread(target=reader) + for _ in range(nreaders)] + with threading_helper.start_threads(threads): + pass + @threading_helper.requires_working_threading() @threading_helper.reap_threads def test_concurrent_whole_buffer_read_and_resize(self): diff --git a/Lib/test/test_free_threading/test_readline.py b/Lib/test/test_free_threading/test_readline.py new file mode 100644 index 000000000000000..f7aa9894031495c --- /dev/null +++ b/Lib/test/test_free_threading/test_readline.py @@ -0,0 +1,59 @@ +import unittest + +from test.support import import_helper +from test.support import threading_helper + +readline = import_helper.import_module("readline") + + +@threading_helper.requires_working_threading() +class TestReadlineRaces(unittest.TestCase): + def test_completer_delims_get_set(self): + def worker(): + for _ in range(100): + readline.get_completer_delims() + readline.set_completer_delims( + ' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?') + readline.set_completer_delims( + ' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?') + readline.get_completer_delims() + + threading_helper.run_concurrently(worker, nthreads=40) + + # get_completer()/get_pre_input_hook() must take the module critical + # section like their setters do; otherwise reading and Py_NewRef-ing the + # stored hook races the setter replacing it (gh-153291). + + def test_completer_get_set(self): + def setter(): + for _ in range(1000): + readline.set_completer(lambda text, state: None) + readline.set_completer(None) + + def getter(): + for _ in range(1000): + readline.get_completer() + + original = readline.get_completer() + self.addCleanup(readline.set_completer, original) + threading_helper.run_concurrently([setter] * 2 + [getter] * 6) + + @unittest.skipUnless(hasattr(readline, "set_pre_input_hook"), + "needs readline.set_pre_input_hook") + def test_pre_input_hook_get_set(self): + def setter(): + for _ in range(1000): + readline.set_pre_input_hook(lambda: None) + readline.set_pre_input_hook(None) + + def getter(): + for _ in range(1000): + readline.get_pre_input_hook() + + original = readline.get_pre_input_hook() + self.addCleanup(readline.set_pre_input_hook, original) + threading_helper.run_concurrently([setter] * 2 + [getter] * 6) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 0c99bbc759e672a..f17d12532c63ced 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -4,8 +4,10 @@ from concurrent.futures import ThreadPoolExecutor from threading import Barrier, Thread from unittest import TestCase +import sys +from test.support import import_helper, threading_helper -from test.support import threading_helper +_testinternalcapi = import_helper.import_module("_testinternalcapi") @@ -84,6 +86,24 @@ def reader_func(): self.run_one(writer_func, reader_func) + def test_attr_cache_mortal(self): + class C: + x = object() + + class D(C): + pass + + def writer_func(): + for _ in range(3000): + C.x = object() + + def reader_func(): + for _ in range(3000): + C.x + D.x + + self.run_one(writer_func, reader_func) + def test___class___modification(self): loops = 200 @@ -160,6 +180,51 @@ def reader(): self.run_one(writer, reader) + def test_per_type_cache_concurrent_reads(self): + class C: + pass + + names = [sys.intern(f"attr_{i}") for i in range( + _testinternalcapi._Py_TYPECACHE_MINSIZE * 4)] + for name in names: + setattr(C, name, name) + # Prime the cache. + for name in names: + getattr(C, name) + + lookup = _testinternalcapi.type_cache_lookup + + def reader(): + for _ in range(500): + for name in names: + hit, value, _ = lookup(C, name) + self.assertEqual(hit, 1, name) + self.assertEqual(value, name) + + threading_helper.run_concurrently(reader, nthreads=NTHREADS) + + def test_per_type_cache_concurrent_invalidate(self): + class C: + x = "value" + + # Prime the cache. + C.x + hit, value, version = _testinternalcapi.type_cache_lookup(C, "x") + self.assertEqual(hit, 1) + self.assertIs(value, "value") + self.assertGreater(version, 0) + + def reader(): + for _ in range(10_000): + self.assertIs(C.x, "value") + + def invalidator(): + for _ in range(10_000): + _testinternalcapi.type_cache_invalidate(C) + + workers = [invalidator] + [reader] * (NTHREADS - 1) + threading_helper.run_concurrently(workers) + def test_race_type_attr_added(self): NROUNDS = 50 NSTOPPERS = 4 diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index c30386afe41849b..b46a1ce6d634a5f 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -438,7 +438,7 @@ def test_setstate_subclasses(self): self.assertIs(type(r[0]), tuple) @support.skip_if_sanitizer("thread sanitizer crashes in __tsan::FuncEntry", thread=True) - @support.skip_if_unlimited_stack_size + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() def test_recursive_pickle(self): with replaced_module('functools', self.module): @@ -579,6 +579,40 @@ def f(**kwargs): with self.assertRaises(RuntimeError): result = p(**{BadStr("poison"): "new_value"}) + def test_call_safety_against_reentrant_mutation(self): + def old_function(*args, **kwargs): + return "old_function", args, kwargs + + def new_function(*args, **kwargs): + return "new_function", args, kwargs + + g_partial = None + + class EvilKey(str): + armed = False + def __hash__(self): + if EvilKey.armed and g_partial is not None: + EvilKey.armed = False + new_args_tuple = ("new_arg",) + new_keywords_dict = {"new_keyword": None} + new_tuple_state = (new_function, new_args_tuple, new_keywords_dict, None) + g_partial.__setstate__(new_tuple_state) + gc.collect() + return str.__hash__(self) + + g_partial = functools.partial(old_function, "old_arg", old_keyword=None) + + kwargs = {EvilKey("evil_key"): None} + EvilKey.armed = True + + result = g_partial(**kwargs) + expected = ("old_function", ("old_arg",), {"old_keyword": None, "evil_key": None}) + self.assertEqual(result, expected) + + result = g_partial() + expected = ("new_function", ("new_arg",), {"new_keyword": None}) + self.assertEqual(result, expected) + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialC(TestPartial, unittest.TestCase): if c_functools: diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 3fc084ea6e9c6e9..4c721c01a34f070 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1434,6 +1434,11 @@ def test_refcount_errors(self): import subprocess code = textwrap.dedent(''' from test.support import gc_collect, SuppressCrashReport + import ctypes.util + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def Py_DecRef(o: ctypes.py_object) -> None: + pass a = [1, 2, 3] b = [a, a] @@ -1445,8 +1450,7 @@ def test_refcount_errors(self): # Simulate the refcount of "a" being too low (compared to the # references held on it by live data), but keeping it above zero # (to avoid deallocating it): - import ctypes - ctypes.pythonapi.Py_DecRef(ctypes.py_object(a)) + Py_DecRef(a) del a del b diff --git a/Lib/test/test_gdb/test_pretty_print.py b/Lib/test/test_gdb/test_pretty_print.py index db3064e3df54c24..edb79ef86d31db5 100644 --- a/Lib/test/test_gdb/test_pretty_print.py +++ b/Lib/test/test_gdb/test_pretty_print.py @@ -114,29 +114,28 @@ def test_bytes(self): @support.requires_resource('cpu') def test_strings(self): 'Verify the pretty-printing of unicode strings' - # We cannot simply call locale.getpreferredencoding() here, - # as GDB might have been linked against a different version - # of Python with a different encoding and coercion policy - # with respect to PEP 538 and PEP 540. + # gdb emits its output in the host charset, which is not necessarily the + # getpreferredencoding() of the (possibly differently coerced) embedded + # Python. stdout, stderr = run_gdb( '--eval-command', - 'python import locale; print(locale.getpreferredencoding())') + 'python import gdb; print(gdb.host_charset())') - encoding = stdout + encoding = stdout.strip() if stderr or not encoding: raise RuntimeError( - f'unable to determine the Python locale preferred encoding ' - f'of embedded Python in GDB\n' + f'unable to determine the host charset of gdb\n' f'stdout={stdout!r}\n' f'stderr={stderr!r}') def check_repr(text): try: text.encode(encoding) - except UnicodeEncodeError: + # LookupError or ValueError if the host charset is unknown or invalid. + except (UnicodeEncodeError, LookupError, ValueError): self.assertGdbRepr(text, ascii(text)) else: - self.assertGdbRepr(text) + self.assertGdbRepr(text, repr(text).encode(encoding).decode('ascii', 'surrogateescape')) self.assertGdbRepr('') self.assertGdbRepr('And now for something hopefully the same') diff --git a/Lib/test/test_gdb/util.py b/Lib/test/test_gdb/util.py index d903adcf2903f34..29c7e9ee8f05498 100644 --- a/Lib/test/test_gdb/util.py +++ b/Lib/test/test_gdb/util.py @@ -78,7 +78,7 @@ def run_gdb(*args, exitcode=0, check=True, **env_vars): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding="utf8", errors="backslashreplace", + encoding="ascii", errors="surrogateescape", env=env) stdout = proc.stdout diff --git a/Lib/test/test_getopt.py b/Lib/test/test_getopt.py index 8d0d5084abbb597..fac2ca3db81d703 100644 --- a/Lib/test/test_getopt.py +++ b/Lib/test/test_getopt.py @@ -149,6 +149,18 @@ def test_getopt(self): ('-a', ''), ('--alpha', '')]) self.assertEqual(args, ['arg1', 'arg2']) + # Allow string for single long argument + opts, args = getopt.getopt(cmdline, 'a::', 'alpha=?') + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2'), ('--alpha', ''), + ('-a', ''), ('--alpha', '')]) + self.assertEqual(args, ['arg1', 'arg2']) + + # Pass everything after -- as args + cmdline = ['-a1', '--alpha=2', '--', '-b', '--beta=5'] + opts, args = getopt.getopt(cmdline, 'a:b', ['alpha=', 'beta']) + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2')]) + self.assertEqual(args, ['-b', '--beta=5']) + self.assertError(getopt.getopt, cmdline, 'a:b', ['alpha', 'beta']) def test_gnu_getopt(self): @@ -191,6 +203,25 @@ def test_gnu_getopt(self): self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', '--beta', '3', 'arg2']) + # Allow string for single long argument + opts, args = getopt.gnu_getopt(cmdline, 'ab:', 'alpha') + self.assertEqual(opts, [('-a', '')]) + self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', + '--beta', '3', 'arg2']) + + # Pass everything after -- as args + cmdline = ['-a1', '--alpha=2', '--', '-b', '--beta=5'] + opts, args = getopt.gnu_getopt(cmdline, 'a:b', ['alpha=', 'beta']) + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2')]) + self.assertEqual(args, ['-b', '--beta=5']) + + # In order arguments + cmdline = ["gamma", "--alpha=3"] + opts, args = getopt.gnu_getopt(cmdline, '-', ["alpha="]) + self.assertEqual(opts, [(None, ['gamma']), ('--alpha', '3')]) + self.assertEqual(args, []) + + def test_issue4629(self): longopts, shortopts = getopt.getopt(['--help='], '', ['help=']) self.assertEqual(longopts, [('--help', '')]) @@ -198,6 +229,10 @@ def test_issue4629(self): self.assertEqual(longopts, [('--help', 'x')]) self.assertRaises(getopt.GetoptError, getopt.getopt, ['--help='], '', ['help']) + def test_getopt_error_str(self): + error = getopt.GetoptError('option -a not recognized', 'a') + self.assertEqual(str(error), 'option -a not recognized') + def test_libref_examples(): """ Examples from the Library Reference: Doc/lib/libgetopt.tex diff --git a/Lib/test/test_importlib/extension/test_finder.py b/Lib/test/test_importlib/extension/test_finder.py index dc77fa78a203fdb..e33c0465654607f 100644 --- a/Lib/test/test_importlib/extension/test_finder.py +++ b/Lib/test/test_importlib/extension/test_finder.py @@ -5,6 +5,7 @@ import unittest import sys +import sysconfig class FinderTests(abc.FinderTests): @@ -61,6 +62,7 @@ def test_failure(self): def test_abi3_extension_suffixes(self): suffixes = self.machinery.EXTENSION_SUFFIXES + platform = sysconfig.get_config_var("SOABI_PLATFORM") if 'win32' in sys.platform: # Either "_d.pyd" or ".pyd" must be in suffixes self.assertTrue({"_d.pyd", ".pyd"}.intersection(suffixes)) @@ -73,6 +75,13 @@ def test_abi3_extension_suffixes(self): self.assertIn(".abi3.so", suffixes) self.assertIn(".abi3t.so", suffixes) + if platform: + if Py_GIL_DISABLED: + self.assertNotIn(f".abi3-{platform}.so", suffixes) + else: + self.assertIn(f".abi3-{platform}.so", suffixes) + self.assertIn(f".abi3t-{platform}.so", suffixes) + (Frozen_FinderTests, Source_FinderTests diff --git a/Lib/test/test_io/utils.py b/Lib/test/test_io/utils.py index 3b1faec2140fbc6..dde49337a24f0b9 100644 --- a/Lib/test/test_io/utils.py +++ b/Lib/test/test_io/utils.py @@ -8,12 +8,13 @@ try: - import ctypes + import ctypes.util except ImportError: def byteslike(*pos, **kw): return array.array("b", bytes(*pos, **kw)) else: - class EmptyStruct(ctypes.Structure): + @ctypes.util.struct + class EmptyStruct: pass def byteslike(*pos, **kw): diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index d97535ba46e677f..16d879a3831bc65 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -263,6 +263,7 @@ def test_subclass_tuple(self): self.assertEqual(True, issubclass(int, (int, (float, int)))) self.assertEqual(True, issubclass(str, (str, (Child, str)))) + @support.skip_if_huge_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() def test_subclass_recursion_limit(self): @@ -270,6 +271,7 @@ def test_subclass_recursion_limit(self): # blown self.assertRaises(RecursionError, blowstack, issubclass, str, str) + @support.skip_if_huge_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() def test_isinstance_recursion_limit(self): @@ -317,7 +319,7 @@ def __bases__(self): self.assertRaises(RecursionError, issubclass, int, X()) self.assertRaises(RecursionError, isinstance, 1, X()) - @support.skip_if_unlimited_stack_size + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_infinite_recursion_via_bases_tuple(self): @@ -329,7 +331,7 @@ def __getattr__(self, attr): with self.assertRaises(RecursionError): issubclass(Failure(), int) - @support.skip_if_unlimited_stack_size + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_infinite_cycle_in_bases(self): diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 1d51fb2de0e69e4..7bff16ddf2155e9 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -1,4 +1,5 @@ import decimal +import unittest.mock from io import StringIO from collections import OrderedDict from test.test_json import PyTest, CTest @@ -47,6 +48,26 @@ def test_empty_objects(self): self.assertEqual(self.loads('[]'), []) self.assertEqual(self.loads('""'), "") + def test_object_hook(self): + s = '{"a":{"b":{}}}' + + expected_result = {"a":{"b":{"x":1}, "x":1}, "x":1} + expected_hook_arguments = [ + {}, {"b": {"x":1}}, {"a": {"b": {"x":1}, "x":1}} + ] + + hook_arguments = [] + + def hook(x): + hook_arguments.append(x) + return {**x, "x":1} + + result = self.loads(s, object_hook=hook) + + self.assertEqual(result, expected_result) + self.assertEqual(hook_arguments, expected_hook_arguments) + + def test_object_pairs_hook(self): s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), @@ -148,6 +169,10 @@ def test_negative_index(self): d = self.json.JSONDecoder() self.assertRaises(ValueError, d.raw_decode, 'a'*42, -50000) + def test_unterminated_string(self): + d = self.json.JSONDecoder() + self.assertRaises(self.JSONDecodeError, d.raw_decode, '"\\') + def test_limit_int(self): maxdigits = 5000 with support.adjust_int_max_str_digits(maxdigits): @@ -155,6 +180,15 @@ def test_limit_int(self): with self.assertRaises(ValueError): self.loads('1' * (maxdigits + 1)) + def test_explicit_cls_skips_json_decoder_default(self): + class CustomDecoder: + pass + + with unittest.mock.patch.object( + CustomDecoder, 'decode', create=True) as mock_decode: + self.loads('{}', cls=CustomDecoder) + mock_decode.assert_called_once() + class TestPyDecode(TestDecode, PyTest): pass class TestCDecode(TestDecode, CTest): pass diff --git a/Lib/test/test_json/test_pickleable.py b/Lib/test/test_json/test_pickleable.py new file mode 100644 index 000000000000000..517cfcdd41a3516 --- /dev/null +++ b/Lib/test/test_json/test_pickleable.py @@ -0,0 +1,14 @@ +import json +import pickle +import unittest + +class TestPickleable(unittest.TestCase): + def test_json_decode_error_is_pickleable(self): + e = json.JSONDecodeError(msg="abc", doc="def", pos=7) + + pickled = pickle.dumps(e) + unpickled = pickle.loads(pickled) + + self.assertEqual(unpickled.msg, e.msg) + self.assertEqual(unpickled.doc, e.doc) + self.assertEqual(unpickled.pos, e.pos) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index d732fc80cf1cf30..cbae9fbb4d624ba 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -69,7 +69,7 @@ def default(self, o): @support.skip_if_pgo_task # fails during PGO training w/ some stack sizes - @support.skip_if_unlimited_stack_size + @support.skip_if_huge_c_stack(500_000) @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_highly_nested_objects_decoding(self): @@ -102,7 +102,7 @@ def test_highly_nested_objects_encoding(self): with support.infinite_recursion(5000): self.dumps(d) - @support.skip_if_unlimited_stack_size + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_endless_recursion(self): diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py deleted file mode 100644 index 795890bd7e4b473..000000000000000 --- a/Lib/test/test_launcher.py +++ /dev/null @@ -1,804 +0,0 @@ -import contextlib -import itertools -import os -import re -import shutil -import subprocess -import sys -import sysconfig -import tempfile -import unittest -from pathlib import Path -from test import support - -if sys.platform != "win32": - raise unittest.SkipTest("test only applies to Windows") - -# Get winreg after the platform check -import winreg - - -PY_EXE = "py.exe" -DEBUG_BUILD = False -if sys.executable.casefold().endswith("_d.exe".casefold()): - PY_EXE = "py_d.exe" - DEBUG_BUILD = True - -# Registry data to create. On removal, everything beneath top-level names will -# be deleted. -TEST_DATA = { - "PythonTestSuite": { - "DisplayName": "Python Test Suite", - "SupportUrl": "https://www.python.org/", - "3.100": { - "DisplayName": "X.Y version", - "InstallPath": { - None: sys.prefix, - "ExecutablePath": "X.Y.exe", - } - }, - "3.100-32": { - "DisplayName": "X.Y-32 version", - "InstallPath": { - None: sys.prefix, - "ExecutablePath": "X.Y-32.exe", - } - }, - "3.100-arm64": { - "DisplayName": "X.Y-arm64 version", - "InstallPath": { - None: sys.prefix, - "ExecutablePath": "X.Y-arm64.exe", - "ExecutableArguments": "-X fake_arg_for_test", - } - }, - "ignored": { - "DisplayName": "Ignored because no ExecutablePath", - "InstallPath": { - None: sys.prefix, - } - }, - }, - "PythonTestSuite1": { - "DisplayName": "Python Test Suite Single", - "3.100": { - "DisplayName": "Single Interpreter", - "InstallPath": { - None: sys.prefix, - "ExecutablePath": sys.executable, - } - } - }, -} - - -TEST_PY_ENV = dict( - PY_PYTHON="PythonTestSuite/3.100", - PY_PYTHON2="PythonTestSuite/3.100-32", - PY_PYTHON3="PythonTestSuite/3.100-arm64", -) - - -TEST_PY_DEFAULTS = "\n".join([ - "[defaults]", - *[f"{k[3:].lower()}={v}" for k, v in TEST_PY_ENV.items()], -]) - - -TEST_PY_COMMANDS = "\n".join([ - "[commands]", - "test-command=TEST_EXE.exe", -]) - - -def quote(s): - s = str(s) - return f'"{s}"' if " " in s else s - - -def create_registry_data(root, data): - def _create_registry_data(root, key, value): - if isinstance(value, dict): - # For a dict, we recursively create keys - with winreg.CreateKeyEx(root, key) as hkey: - for k, v in value.items(): - _create_registry_data(hkey, k, v) - elif isinstance(value, str): - # For strings, we set values. 'key' may be None in this case - winreg.SetValueEx(root, key, None, winreg.REG_SZ, value) - else: - raise TypeError("don't know how to create data for '{}'".format(value)) - - for k, v in data.items(): - _create_registry_data(root, k, v) - - -def enum_keys(root): - for i in itertools.count(): - try: - yield winreg.EnumKey(root, i) - except OSError as ex: - if ex.winerror == 259: - break - raise - - -def delete_registry_data(root, keys): - ACCESS = winreg.KEY_WRITE | winreg.KEY_ENUMERATE_SUB_KEYS - for key in list(keys): - with winreg.OpenKey(root, key, access=ACCESS) as hkey: - delete_registry_data(hkey, enum_keys(hkey)) - winreg.DeleteKey(root, key) - - -def is_installed(tag): - key = rf"Software\Python\PythonCore\{tag}\InstallPath" - for root, flag in [ - (winreg.HKEY_CURRENT_USER, 0), - (winreg.HKEY_LOCAL_MACHINE, winreg.KEY_WOW64_64KEY), - (winreg.HKEY_LOCAL_MACHINE, winreg.KEY_WOW64_32KEY), - ]: - try: - winreg.CloseKey(winreg.OpenKey(root, key, access=winreg.KEY_READ | flag)) - return True - except OSError: - pass - return False - - -class PreservePyIni: - def __init__(self, path, content): - self.path = Path(path) - self.content = content - self._preserved = None - - def __enter__(self): - try: - self._preserved = self.path.read_bytes() - except FileNotFoundError: - self._preserved = None - self.path.write_text(self.content, encoding="utf-16") - - def __exit__(self, *exc_info): - if self._preserved is None: - self.path.unlink() - else: - self.path.write_bytes(self._preserved) - - -class RunPyMixin: - py_exe = None - - @classmethod - def find_py(cls): - py_exe = None - if sysconfig.is_python_build(): - py_exe = Path(sys.executable).parent / PY_EXE - else: - for p in os.getenv("PATH").split(";"): - if p: - py_exe = Path(p) / PY_EXE - if py_exe.is_file(): - break - else: - py_exe = None - - # Test launch and check version, to exclude installs of older - # releases when running outside of a source tree - if py_exe: - try: - with subprocess.Popen( - [py_exe, "-h"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="ascii", - errors="ignore", - ) as p: - p.stdin.close() - version = next(p.stdout, "\n").splitlines()[0].rpartition(" ")[2] - p.stdout.read() - p.wait(10) - if not sys.version.startswith(version): - py_exe = None - except OSError: - py_exe = None - - if not py_exe: - raise unittest.SkipTest( - "cannot locate '{}' for test".format(PY_EXE) - ) - return py_exe - - def get_py_exe(self): - if not self.py_exe: - self.py_exe = self.find_py() - return self.py_exe - - def run_py(self, args, env=None, allow_fail=False, expect_returncode=0, argv=None): - if not self.py_exe: - self.py_exe = self.find_py() - - ignore = {"VIRTUAL_ENV", "PY_PYTHON", "PY_PYTHON2", "PY_PYTHON3"} - env = { - **{k.upper(): v for k, v in os.environ.items() if k.upper() not in ignore}, - "PYLAUNCHER_DEBUG": "1", - "PYLAUNCHER_DRYRUN": "1", - "PYLAUNCHER_LIMIT_TO_COMPANY": "", - **{k.upper(): v for k, v in (env or {}).items()}, - } - if ini_dir := getattr(self, '_ini_dir', None): - env.setdefault("_PYLAUNCHER_INIDIR", ini_dir) - if not argv: - argv = [self.py_exe, *args] - with subprocess.Popen( - argv, - env=env, - executable=self.py_exe, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) as p: - p.stdin.close() - p.wait(10) - out = p.stdout.read().decode("utf-8", "replace") - err = p.stderr.read().decode("ascii", "replace").replace("\uFFFD", "?") - if p.returncode != expect_returncode and support.verbose and not allow_fail: - print("++ COMMAND ++") - print([self.py_exe, *args]) - print("++ STDOUT ++") - print(out) - print("++ STDERR ++") - print(err) - if allow_fail and p.returncode != expect_returncode: - raise subprocess.CalledProcessError(p.returncode, [self.py_exe, *args], out, err) - else: - self.assertEqual(expect_returncode, p.returncode) - data = { - s.partition(":")[0]: s.partition(":")[2].lstrip() - for s in err.splitlines() - if not s.startswith("#") and ":" in s - } - data["stdout"] = out - data["stderr"] = err - return data - - def py_ini(self, content): - ini_dir = getattr(self, '_ini_dir', None) - if not ini_dir: - local_appdata = os.environ.get("LOCALAPPDATA") - if not local_appdata: - raise unittest.SkipTest("LOCALAPPDATA environment variable is " - "missing or empty") - ini_dir = local_appdata - return PreservePyIni(Path(ini_dir) / "py.ini", content) - - @contextlib.contextmanager - def script(self, content, encoding="utf-8"): - file = Path(tempfile.mktemp(dir=os.getcwd()) + ".py") - if isinstance(content, bytes): - file.write_bytes(content) - else: - file.write_text(content, encoding=encoding) - try: - yield file - finally: - file.unlink() - - @contextlib.contextmanager - def fake_venv(self): - venv = Path.cwd() / "Scripts" - venv.mkdir(exist_ok=True, parents=True) - venv_exe = (venv / ("python_d.exe" if DEBUG_BUILD else "python.exe")) - venv_exe.touch() - try: - yield venv_exe, {"VIRTUAL_ENV": str(venv.parent)} - finally: - shutil.rmtree(venv) - - -class TestLauncher(unittest.TestCase, RunPyMixin): - @classmethod - def setUpClass(cls): - with winreg.CreateKey(winreg.HKEY_CURRENT_USER, rf"Software\Python") as key: - create_registry_data(key, TEST_DATA) - - if support.verbose: - p = subprocess.check_output("reg query HKCU\\Software\\Python /s") - #print(p.decode('mbcs')) - - cls._ini_dir = tempfile.mkdtemp() - cls.addClassCleanup(shutil.rmtree, cls._ini_dir, ignore_errors=True) - - @classmethod - def tearDownClass(cls): - with winreg.OpenKey(winreg.HKEY_CURRENT_USER, rf"Software\Python", access=winreg.KEY_WRITE | winreg.KEY_ENUMERATE_SUB_KEYS) as key: - delete_registry_data(key, TEST_DATA) - - - def test_version(self): - data = self.run_py(["-0"]) - self.assertEqual(self.py_exe, Path(data["argv0"])) - self.assertEqual(sys.version.partition(" ")[0], data["version"]) - - def test_help_option(self): - data = self.run_py(["-h"]) - self.assertEqual("True", data["SearchInfo.help"]) - - def test_list_option(self): - for opt, v1, v2 in [ - ("-0", "True", "False"), - ("-0p", "False", "True"), - ("--list", "True", "False"), - ("--list-paths", "False", "True"), - ]: - with self.subTest(opt): - data = self.run_py([opt]) - self.assertEqual(v1, data["SearchInfo.list"]) - self.assertEqual(v2, data["SearchInfo.listPaths"]) - - def test_list(self): - data = self.run_py(["--list"]) - found = {} - expect = {} - for line in data["stdout"].splitlines(): - m = re.match(r"\s*(.+?)\s+?(\*\s+)?(.+)$", line) - if m: - found[m.group(1)] = m.group(3) - for company in TEST_DATA: - company_data = TEST_DATA[company] - tags = [t for t in company_data if isinstance(company_data[t], dict)] - for tag in tags: - arg = f"-V:{company}/{tag}" - expect[arg] = company_data[tag]["DisplayName"] - expect.pop(f"-V:{company}/ignored", None) - - actual = {k: v for k, v in found.items() if k in expect} - try: - self.assertDictEqual(expect, actual) - except: - if support.verbose: - print("*** STDOUT ***") - print(data["stdout"]) - raise - - def test_list_paths(self): - data = self.run_py(["--list-paths"]) - found = {} - expect = {} - for line in data["stdout"].splitlines(): - m = re.match(r"\s*(.+?)\s+?(\*\s+)?(.+)$", line) - if m: - found[m.group(1)] = m.group(3) - for company in TEST_DATA: - company_data = TEST_DATA[company] - tags = [t for t in company_data if isinstance(company_data[t], dict)] - for tag in tags: - arg = f"-V:{company}/{tag}" - install = company_data[tag]["InstallPath"] - try: - expect[arg] = install["ExecutablePath"] - try: - expect[arg] += " " + install["ExecutableArguments"] - except KeyError: - pass - except KeyError: - expect[arg] = str(Path(install[None]) / Path(sys.executable).name) - - expect.pop(f"-V:{company}/ignored", None) - - actual = {k: v for k, v in found.items() if k in expect} - try: - self.assertDictEqual(expect, actual) - except: - if support.verbose: - print("*** STDOUT ***") - print(data["stdout"]) - raise - - def test_filter_to_company(self): - company = "PythonTestSuite" - data = self.run_py([f"-V:{company}/"]) - self.assertEqual("X.Y.exe", data["LaunchCommand"]) - self.assertEqual(company, data["env.company"]) - self.assertEqual("3.100", data["env.tag"]) - - def test_filter_to_company_with_default(self): - company = "PythonTestSuite" - data = self.run_py([f"-V:{company}/"], env=dict(PY_PYTHON="3.0")) - self.assertEqual("X.Y.exe", data["LaunchCommand"]) - self.assertEqual(company, data["env.company"]) - self.assertEqual("3.100", data["env.tag"]) - - def test_filter_to_tag(self): - company = "PythonTestSuite" - data = self.run_py(["-V:3.100"]) - self.assertEqual("X.Y.exe", data["LaunchCommand"]) - self.assertEqual(company, data["env.company"]) - self.assertEqual("3.100", data["env.tag"]) - - data = self.run_py(["-V:3.100-32"]) - self.assertEqual("X.Y-32.exe", data["LaunchCommand"]) - self.assertEqual(company, data["env.company"]) - self.assertEqual("3.100-32", data["env.tag"]) - - data = self.run_py(["-V:3.100-arm64"]) - self.assertEqual("X.Y-arm64.exe -X fake_arg_for_test", data["LaunchCommand"]) - self.assertEqual(company, data["env.company"]) - self.assertEqual("3.100-arm64", data["env.tag"]) - - def test_filter_to_company_and_tag(self): - company = "PythonTestSuite" - data = self.run_py([f"-V:{company}/3.1"], expect_returncode=103) - - data = self.run_py([f"-V:{company}/3.100"]) - self.assertEqual("X.Y.exe", data["LaunchCommand"]) - self.assertEqual(company, data["env.company"]) - self.assertEqual("3.100", data["env.tag"]) - - def test_filter_with_single_install(self): - company = "PythonTestSuite1" - data = self.run_py( - ["-V:Nonexistent"], - env={"PYLAUNCHER_LIMIT_TO_COMPANY": company}, - expect_returncode=103, - ) - - def test_search_major_3(self): - try: - data = self.run_py(["-3"], allow_fail=True) - except subprocess.CalledProcessError: - raise unittest.SkipTest("requires at least one Python 3.x install") - self.assertEqual("PythonCore", data["env.company"]) - self.assertStartsWith(data["env.tag"], "3.") - - def test_search_major_3_32(self): - try: - data = self.run_py(["-3-32"], allow_fail=True) - except subprocess.CalledProcessError: - if not any(is_installed(f"3.{i}-32") for i in range(5, 11)): - raise unittest.SkipTest("requires at least one 32-bit Python 3.x install") - raise - self.assertEqual("PythonCore", data["env.company"]) - self.assertStartsWith(data["env.tag"], "3.") - self.assertEndsWith(data["env.tag"], "-32") - - def test_search_major_2(self): - try: - data = self.run_py(["-2"], allow_fail=True) - except subprocess.CalledProcessError: - if not is_installed("2.7"): - raise unittest.SkipTest("requires at least one Python 2.x install") - raise - self.assertEqual("PythonCore", data["env.company"]) - self.assertStartsWith(data["env.tag"], "2.") - - def test_py_default(self): - with self.py_ini(TEST_PY_DEFAULTS): - data = self.run_py(["-arg"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual("X.Y.exe -arg", data["stdout"].strip()) - - def test_py2_default(self): - with self.py_ini(TEST_PY_DEFAULTS): - data = self.run_py(["-2", "-arg"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100-32", data["SearchInfo.tag"]) - self.assertEqual("X.Y-32.exe -arg", data["stdout"].strip()) - - def test_py3_default(self): - with self.py_ini(TEST_PY_DEFAULTS): - data = self.run_py(["-3", "-arg"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) - self.assertEqual("X.Y-arm64.exe -X fake_arg_for_test -arg", data["stdout"].strip()) - - def test_py_default_env(self): - data = self.run_py(["-arg"], env=TEST_PY_ENV) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual("X.Y.exe -arg", data["stdout"].strip()) - - def test_py2_default_env(self): - data = self.run_py(["-2", "-arg"], env=TEST_PY_ENV) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100-32", data["SearchInfo.tag"]) - self.assertEqual("X.Y-32.exe -arg", data["stdout"].strip()) - - def test_py3_default_env(self): - data = self.run_py(["-3", "-arg"], env=TEST_PY_ENV) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) - self.assertEqual("X.Y-arm64.exe -X fake_arg_for_test -arg", data["stdout"].strip()) - - def test_py_default_short_argv0(self): - with self.py_ini(TEST_PY_DEFAULTS): - for argv0 in ['"py.exe"', 'py.exe', '"py"', 'py']: - with self.subTest(argv0): - data = self.run_py(["--version"], argv=f'{argv0} --version') - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual("X.Y.exe --version", data["stdout"].strip()) - - def test_py_default_in_list(self): - data = self.run_py(["-0"], env=TEST_PY_ENV) - default = None - for line in data["stdout"].splitlines(): - m = re.match(r"\s*-V:(.+?)\s+?\*\s+(.+)$", line) - if m: - default = m.group(1) - break - self.assertEqual("PythonTestSuite/3.100", default) - - def test_virtualenv_in_list(self): - with self.fake_venv() as (venv_exe, env): - data = self.run_py(["-0p"], env=env) - for line in data["stdout"].splitlines(): - m = re.match(r"\s*\*\s+(.+)$", line) - if m: - self.assertEqual(str(venv_exe), m.group(1)) - break - else: - if support.verbose: - print(data["stdout"]) - print(data["stderr"]) - self.fail("did not find active venv path") - - data = self.run_py(["-0"], env=env) - for line in data["stdout"].splitlines(): - m = re.match(r"\s*\*\s+(.+)$", line) - if m: - self.assertEqual("Active venv", m.group(1)) - break - else: - self.fail("did not find active venv entry") - - def test_virtualenv_with_env(self): - with self.fake_venv() as (venv_exe, env): - data1 = self.run_py([], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) - data2 = self.run_py(["-V:PythonTestSuite/3"], env={**env, "PY_PYTHON": "PythonTestSuite/3"}) - # Compare stdout, because stderr goes via ascii - self.assertEqual(data1["stdout"].strip(), quote(venv_exe)) - self.assertEqual(data1["SearchInfo.lowPriorityTag"], "True") - # Ensure passing the argument doesn't trigger the same behaviour - self.assertNotEqual(data2["stdout"].strip(), quote(venv_exe)) - self.assertNotEqual(data2["SearchInfo.lowPriorityTag"], "True") - - def test_py_shebang(self): - with self.py_ini(TEST_PY_DEFAULTS): - with self.script("#! /usr/bin/python -prearg") as script: - data = self.run_py([script, "-postarg"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) - - def test_python_shebang(self): - with self.py_ini(TEST_PY_DEFAULTS): - with self.script("#! python -prearg") as script: - data = self.run_py([script, "-postarg"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) - - def test_py2_shebang(self): - with self.py_ini(TEST_PY_DEFAULTS): - with self.script("#! /usr/bin/python2 -prearg") as script: - data = self.run_py([script, "-postarg"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100-32", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", - data["stdout"].strip()) - - def test_py3_shebang(self): - with self.py_ini(TEST_PY_DEFAULTS): - with self.script("#! /usr/bin/python3 -prearg") as script: - data = self.run_py([script, "-postarg"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", - data["stdout"].strip()) - - def test_py_shebang_nl(self): - with self.py_ini(TEST_PY_DEFAULTS): - with self.script("#! /usr/bin/python -prearg\n") as script: - data = self.run_py([script, "-postarg"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", - data["stdout"].strip()) - - def test_py2_shebang_nl(self): - with self.py_ini(TEST_PY_DEFAULTS): - with self.script("#! /usr/bin/python2 -prearg\n") as script: - data = self.run_py([script, "-postarg"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100-32", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-32.exe -prearg {quote(script)} -postarg", - data["stdout"].strip()) - - def test_py3_shebang_nl(self): - with self.py_ini(TEST_PY_DEFAULTS): - with self.script("#! /usr/bin/python3 -prearg\n") as script: - data = self.run_py([script, "-postarg"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100-arm64", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y-arm64.exe -X fake_arg_for_test -prearg {quote(script)} -postarg", - data["stdout"].strip()) - - def test_py_shebang_short_argv0(self): - with self.py_ini(TEST_PY_DEFAULTS): - with self.script("#! /usr/bin/python -prearg") as script: - # Override argv to only pass "py.exe" as the command - data = self.run_py([script, "-postarg"], argv=f'"py.exe" "{script}" -postarg') - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f'X.Y.exe -prearg "{script}" -postarg', data["stdout"].strip()) - - def test_py_shebang_valid_bom(self): - with self.py_ini(TEST_PY_DEFAULTS): - content = "#! /usr/bin/python -prearg".encode("utf-8") - with self.script(b"\xEF\xBB\xBF" + content) as script: - data = self.run_py([script, "-postarg"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe -prearg {quote(script)} -postarg", data["stdout"].strip()) - - def test_py_shebang_invalid_bom(self): - with self.py_ini(TEST_PY_DEFAULTS): - content = "#! /usr/bin/python3 -prearg".encode("utf-8") - with self.script(b"\xEF\xAA\xBF" + content) as script: - data = self.run_py([script, "-postarg"]) - self.assertIn("Invalid BOM", data["stderr"]) - self.assertEqual("PythonTestSuite", data["SearchInfo.company"]) - self.assertEqual("3.100", data["SearchInfo.tag"]) - self.assertEqual(f"X.Y.exe {quote(script)} -postarg", data["stdout"].strip()) - - def test_py_handle_64_in_ini(self): - with self.py_ini("\n".join(["[defaults]", "python=3.999-64"])): - # Expect this to fail, but should get oldStyleTag flipped on - data = self.run_py([], allow_fail=True, expect_returncode=103) - self.assertEqual("3.999-64", data["SearchInfo.tag"]) - self.assertEqual("True", data["SearchInfo.oldStyleTag"]) - - def test_search_path(self): - exe = Path("arbitrary-exe-name.exe").absolute() - exe.touch() - self.addCleanup(exe.unlink) - with self.py_ini(TEST_PY_DEFAULTS): - with self.script(f"#! /usr/bin/env {exe.stem} -prearg") as script: - data = self.run_py( - [script, "-postarg"], - env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, - ) - self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", - data["stdout"].strip()) - - def test_search_path_exe(self): - # Leave the .exe on the name to ensure we don't add it a second time - exe = Path("arbitrary-exe-name.exe").absolute() - exe.touch() - self.addCleanup(exe.unlink) - with self.py_ini(TEST_PY_DEFAULTS): - with self.script(f"#! /usr/bin/env {exe.name} -prearg") as script: - data = self.run_py( - [script, "-postarg"], - env={"PATH": f"{exe.parent};{os.getenv('PATH')}"}, - ) - self.assertEqual(f"{quote(exe)} -prearg {quote(script)} -postarg", - data["stdout"].strip()) - - def test_recursive_search_path(self): - stem = self.get_py_exe().stem - with self.py_ini(TEST_PY_DEFAULTS): - with self.script(f"#! /usr/bin/env {stem}") as script: - data = self.run_py( - [script], - env={"PATH": f"{self.get_py_exe().parent};{os.getenv('PATH')}"}, - ) - # The recursive search is ignored and we get normal "py" behavior - self.assertEqual(f"X.Y.exe {quote(script)}", data["stdout"].strip()) - - def test_install(self): - data = self.run_py(["-V:3.10"], env={"PYLAUNCHER_ALWAYS_INSTALL": "1"}, expect_returncode=111) - cmd = data["stdout"].strip() - # If winget is runnable, we should find it. Otherwise, we'll be trying - # to open the Store. - try: - subprocess.check_call(["winget.exe", "--version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - except FileNotFoundError: - self.assertIn("ms-windows-store://", cmd) - else: - self.assertIn("winget.exe", cmd) - # Both command lines include the store ID - self.assertIn("9PJPW5LDXLZ5", cmd) - - def test_literal_shebang_absolute(self): - with self.script("#! C:/some_random_app -witharg") as script: - data = self.run_py([script]) - self.assertEqual( - f"C:\\some_random_app -witharg {quote(script)}", - data["stdout"].strip(), - ) - - def test_literal_shebang_relative(self): - with self.script("#! ..\\some_random_app -witharg") as script: - data = self.run_py([script]) - self.assertEqual( - f"{quote(script.parent.parent / 'some_random_app')} -witharg {quote(script)}", - data["stdout"].strip(), - ) - - def test_literal_shebang_quoted(self): - with self.script('#! "some random app" -witharg') as script: - data = self.run_py([script]) - self.assertEqual( - f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", - data["stdout"].strip(), - ) - - with self.script('#! some" random "app -witharg') as script: - data = self.run_py([script]) - self.assertEqual( - f"{quote(script.parent / 'some random app')} -witharg {quote(script)}", - data["stdout"].strip(), - ) - - def test_literal_shebang_quoted_escape(self): - with self.script('#! some\\" random "app -witharg') as script: - data = self.run_py([script]) - self.assertEqual( - f"{quote(script.parent / 'some/ random app')} -witharg {quote(script)}", - data["stdout"].strip(), - ) - - def test_literal_shebang_command(self): - with self.py_ini(TEST_PY_COMMANDS): - with self.script('#! test-command arg1') as script: - data = self.run_py([script]) - self.assertEqual( - f"TEST_EXE.exe arg1 {quote(script)}", - data["stdout"].strip(), - ) - - def test_literal_shebang_invalid_template(self): - with self.script('#! /usr/bin/not-python arg1') as script: - data = self.run_py([script]) - expect = script.parent / "/usr/bin/not-python" - self.assertEqual( - f"{quote(expect)} arg1 {quote(script)}", - data["stdout"].strip(), - ) - - def test_shebang_command_in_venv(self): - stem = "python-that-is-not-on-path" - - # First ensure that our test name doesn't exist, and the launcher does - # not match any installed env - with self.script(f'#! /usr/bin/env {stem} arg1') as script: - data = self.run_py([script], expect_returncode=103) - - with self.fake_venv() as (venv_exe, env): - # Put a "normal" Python on PATH as a distraction. - # The active VIRTUAL_ENV should be preferred when the name isn't an - # exact match. - exe = Path(Path(venv_exe).name).absolute() - exe.touch() - self.addCleanup(exe.unlink) - env["PATH"] = f"{exe.parent};{os.environ['PATH']}" - - with self.script(f'#! /usr/bin/env {stem} arg1') as script: - data = self.run_py([script], env=env) - self.assertEqual(data["stdout"].strip(), f"{quote(venv_exe)} arg1 {quote(script)}") - - with self.script(f'#! /usr/bin/env {exe.stem} arg1') as script: - data = self.run_py([script], env=env) - self.assertEqual(data["stdout"].strip(), f"{quote(exe)} arg1 {quote(script)}") - - def test_shebang_executable_extension(self): - with self.script('#! /usr/bin/env python3.99') as script: - data = self.run_py([script], expect_returncode=103) - expect = "# Search PATH for python3.99.exe" - actual = [line.strip() for line in data["stderr"].splitlines() - if line.startswith("# Search PATH")] - self.assertEqual([expect], actual) diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index cf1d3eb793a8b94..899ecec1ba2afa4 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -676,52 +676,35 @@ def test_lazy_modules_tracks_lazy_imports(self): @support.requires_subprocess() class ErrorHandlingTests(LazyImportTestCase): - """Tests for error handling during lazy import reification. + """Tests for error handling during lazy import reification.""" - PEP 810: Errors during reification should show exception chaining with - both the lazy import definition location and the access location. - """ - - def test_import_error_shows_chained_traceback(self): + def test_missing_lazy_submodule_raises_attribute_error(self): """Accessing a nonexistent lazy submodule via parent attr raises AttributeError.""" code = textwrap.dedent(""" - import sys lazy import test.test_lazy_import.data.nonexistent_module try: - x = test.test_lazy_import.data.nonexistent_module - except AttributeError as e: - print("OK") + _ = test.test_lazy_import.data.nonexistent_module + except AttributeError: + pass + else: + raise AssertionError("AttributeError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) - def test_attribute_error_on_from_import_shows_chained_traceback(self): + def test_missing_lazy_from_import_shows_chained_traceback(self): """Accessing missing attribute from lazy from-import should chain errors.""" - # Tests 'lazy from module import nonexistent' behavior code = textwrap.dedent(""" - import sys lazy from test.test_lazy_import.data.basic2 import nonexistent_name try: - x = nonexistent_name + _ = nonexistent_name except ImportError as e: - # PEP 810: Enhanced error reporting through exception chaining assert e.__cause__ is not None, "Expected chained exception" - print("OK") + else: + raise AssertionError("ImportError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) def test_reification_retries_on_failure(self): """Failed reification should allow retry on subsequent access. @@ -731,53 +714,92 @@ def test_reification_retries_on_failure(self): """ code = textwrap.dedent(""" import sys - import types lazy import test.test_lazy_import.data.broken_module - # First access - should fail try: - x = test.test_lazy_import.data.broken_module - except AttributeError: - pass + _ = test.test_lazy_import.data.broken_module + except ValueError as exc: + assert str(exc) == "This module always fails to import", exc + else: + raise AssertionError("ValueError was not raised") + + assert "test.test_lazy_import.data.broken_module" not in sys.modules - # The lazy object should still be a lazy proxy (not reified) - g = globals() - lazy_obj = g['test'] - # The root 'test' binding should still allow retry - # Second access - should also fail (retry the import) try: - x = test.test_lazy_import.data.broken_module - except AttributeError: - print("OK - retry worked") + _ = test.test_lazy_import.data.broken_module + except ValueError as exc: + assert str(exc) == "This module always fails to import", exc + else: + raise AssertionError("ValueError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) - def test_error_during_module_execution_propagates(self): - """Errors in module code during reification should propagate correctly.""" + def test_lazy_submodule_traceback_hides_importlib_frames(self): code = textwrap.dedent(""" - import sys + import traceback + lazy import test.test_lazy_import.data.broken_module try: _ = test.test_lazy_import.data.broken_module - print("FAIL - should have raised") - except AttributeError: - print("OK") + except ValueError as exc: + frames = traceback.extract_tb(exc.__traceback__) + assert [frame.name for frame in frames] == ["", ""], frames + assert frames[0].filename == "", frames + assert frames[1].filename.endswith("broken_module.py"), frames + else: + raise AssertionError("ValueError was not raised") """) - result = subprocess.run( - [sys.executable, "-c", code], - capture_output=True, - text=True - ) - self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") - self.assertIn("OK", result.stdout) + assert_python_ok("-c", code) + + def test_module_not_found_during_module_execution_propagates(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.missing_dependency + + try: + _ = test.test_lazy_import.data.missing_dependency + except ModuleNotFoundError as exc: + assert exc.name == "missing_dependency_for_lazy_import_test", exc.name + assert str(exc) == "No module named 'missing_dependency_for_lazy_import_test'", exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_self_named_module_not_found_during_module_execution_propagates(self): + code = textwrap.dedent(""" + lazy import test.test_lazy_import.data.self_named_module_not_found + + try: + _ = test.test_lazy_import.data.self_named_module_not_found + except ModuleNotFoundError as exc: + assert exc.name == "test.test_lazy_import.data.self_named_module_not_found", exc.name + assert str(exc) == "boom", exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) + + def test_none_in_sys_modules_during_submodule_resolution_propagates(self): + code = textwrap.dedent(""" + import sys + + sys.modules["test.test_lazy_import.data.blocked_module"] = None + lazy import test.test_lazy_import.data.blocked_module + + try: + _ = test.test_lazy_import.data.blocked_module + except ModuleNotFoundError as exc: + assert exc.name == "test.test_lazy_import.data.blocked_module", exc.name + assert str(exc) == ( + "import of test.test_lazy_import.data.blocked_module " + "halted; None in sys.modules" + ), exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + """) + assert_python_ok("-c", code) def test_circular_lazy_import_does_not_crash_for_gh_144727(self): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/Lib/test/test_lazy_import/data/missing_dependency.py b/Lib/test/test_lazy_import/data/missing_dependency.py new file mode 100644 index 000000000000000..aa2f2d0f65f73fe --- /dev/null +++ b/Lib/test/test_lazy_import/data/missing_dependency.py @@ -0,0 +1 @@ +import missing_dependency_for_lazy_import_test diff --git a/Lib/test/test_lazy_import/data/self_named_module_not_found.py b/Lib/test/test_lazy_import/data/self_named_module_not_found.py new file mode 100644 index 000000000000000..941230d5da211d8 --- /dev/null +++ b/Lib/test/test_lazy_import/data/self_named_module_not_found.py @@ -0,0 +1 @@ +raise ModuleNotFoundError("boom", name=__name__) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index a7ea128d64055a1..ccc7cce86883c89 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -2138,6 +2138,45 @@ def setUp(self): self.addCleanup(os_helper.unlink, self.address) SysLogHandlerTest.setUp(self) + def test_bytes_address(self): + # The Unix socket address can also be specified as bytes. + if self.server_exception: + self.skipTest(self.server_exception) + hdlr = logging.handlers.SysLogHandler(os.fsencode(self.address)) + self.addCleanup(hdlr.close) + self.assertTrue(hdlr.unixsocket) + logger = logging.getLogger("slh-bytes") + logger.addHandler(hdlr) + self.addCleanup(logger.removeHandler, hdlr) + logger.error("sp\xe4m") + self.handled.wait(support.LONG_TIMEOUT) + self.assertEqual(self.log_output, b'<11>sp\xc3\xa4m\x00') + +@unittest.skipUnless(sys.platform in ('linux', 'android'), + 'Linux specific test') +class AbstractNamespaceSysLogHandlerTest(BaseTest): + + """Test for SysLogHandler with a socket in the abstract namespace.""" + + def check(self, address): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) + self.addCleanup(sock.close) + sock.bind(address) + sock.settimeout(support.LONG_TIMEOUT) + hdlr = logging.handlers.SysLogHandler(address) + self.addCleanup(hdlr.close) + self.assertTrue(hdlr.unixsocket) + hdlr.emit(logging.makeLogRecord({'msg': 'sp\xe4m'})) + self.assertEqual(sock.recv(1024), b'<12>sp\xc3\xa4m\x00') + + def test_str_address(self): + # A str address is encoded with the filesystem encoding. + self.check('\0' + os_helper.TESTFN) + + def test_bytes_address(self): + # The name is an arbitrary byte sequence, it need not be decodable. + self.check(b'\0test_logging_%d_\xff\xfe' % os.getpid()) + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'IPv6 support required for this test.') class IPv6SysLogHandlerTest(SysLogHandlerTest): @@ -3643,14 +3682,14 @@ def setup_via_listener(self, text, verify=None): # Ask for a randomly assigned port (by using port 0) t = logging.config.listen(0, verify) t.start() - t.ready.wait() + self.assertTrue(t.ready.wait(support.LONG_TIMEOUT), + msg='the listener did not start') # Now get the port allocated port = t.port t.ready.clear() try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(2.0) - sock.connect(('localhost', port)) + # The server can listen on IPv6, so do not force a family. + sock = socket.create_connection(('localhost', port), timeout=2.0) slen = struct.pack('>L', len(text)) s = slen + text @@ -3765,6 +3804,18 @@ def verify_reverse(stuff): ('ERROR', '2'), ], pat=r"^[\w.]+ -> (\w+): (\d+)$") + @support.requires_working_socket() + def test_listen_server_error(self): + # The "ready" event should be set even if the server fails to start. + t = logging.config.listen(-1) + t.daemon = True + with threading_helper.catch_threading_exception() as cm: + t.start() + self.assertTrue(t.ready.wait(support.SHORT_TIMEOUT), + msg='the listener did not report the failure') + threading_helper.join_thread(t) + self.assertIs(cm.exc_type, OverflowError) + def test_bad_format(self): self.assertRaises(ValueError, self.apply_config, self.bad_format) @@ -6704,7 +6755,14 @@ def add_record(message: str) -> None: self.assertTrue(found, msg=msg) def test_rollover_at_midnight(self, weekly=False): + # Create the log file in a fresh directory under a never used name: + # on Windows, NTFS file tunneling restores the original creation time + # of a file recreated with the same name. os_helper.unlink(self.fn) + dirname = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, dirname) + self.fn = os.path.join(dirname, 'test_rollover.log') + # Emit the first records a little after the beginning of a whole # second, so that their file times fall inside that second and not the # previous one, which would cause an unwanted rollover. @@ -6730,7 +6788,24 @@ def test_rollover_at_midnight(self, weekly=False): # changed, so the rollover cannot be forced by back-dating the file. # Wait until the clock reaches a rollover time set one second ahead. rollover = int(time.time()) + 1 + if rollover - time.time() < 0.1: + # Leave time to emit a record before the rollover time. + rollover += 1 atTime = datetime.datetime.fromtimestamp(rollover).time() + rolloverDate = (datetime.datetime.fromtimestamp(rollover) + - datetime.timedelta(days=7 if weekly else 1)) + otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' + + # A record emitted before the rollover time is not rolled over. + fh = logging.handlers.TimedRotatingFileHandler( + self.fn, encoding="utf-8", when=when, atTime=atTime) + fh.setFormatter(fmt) + r2 = logging.makeLogRecord({'msg': 'testing1 3'}) + fh.emit(r2) + fh.close() + self.assertFalse(os.path.exists(otherfn), + msg=f'{otherfn} was rolled over too early') + while time.time() < rollover: time.sleep(rollover - time.time()) for i in range(2): @@ -6740,9 +6815,6 @@ def test_rollover_at_midnight(self, weekly=False): r2 = logging.makeLogRecord({'msg': f'testing2 {i}'}) fh.emit(r2) fh.close() - rolloverDate = (datetime.datetime.fromtimestamp(rollover) - - datetime.timedelta(days=7 if weekly else 1)) - otherfn = f'{self.fn}.{rolloverDate:%Y-%m-%d}' self.assertLogFile(otherfn) with open(self.fn, encoding="utf-8") as f: for i, line in enumerate(f): diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 2e2ac147968dd4a..55c6ad04d44d828 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -907,9 +907,10 @@ def test_resize_up_anonymous_mapping(self): with mmap.mmap(-1, start_size) as m: m[:] = data - if sys.platform.startswith(('linux', 'android')): - # Can't expand a shared anonymous mapping on Linux. - # See https://bugzilla.kernel.org/show_bug.cgi?id=8691 + if sys.platform.startswith(('linux', 'android', 'netbsd')): + # Can't expand a shared anonymous mapping on Linux + # (see https://bugzilla.kernel.org/show_bug.cgi?id=8691) + # or NetBSD. with self.assertRaises(ValueError): m.resize(new_size) else: @@ -1175,8 +1176,8 @@ def test_flush_parameters(self): if hasattr(mmap, 'MS_INVALIDATE'): m.flush(PAGESIZE * 2, flags=mmap.MS_INVALIDATE) if hasattr(mmap, 'MS_ASYNC') and hasattr(mmap, 'MS_INVALIDATE'): - if sys.platform == 'freebsd': - # FreeBSD doesn't support this combination + if sys.platform.startswith(('freebsd', 'dragonfly')): + # FreeBSD and DragonFly don't support this combination with self.assertRaises(OSError) as cm: m.flush(0, PAGESIZE, flags=mmap.MS_ASYNC | mmap.MS_INVALIDATE) self.assertEqual(cm.exception.errno, errno.EINVAL) diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py index fef86ce323e54d5..a588b0ff46014a1 100644 --- a/Lib/test/test_msvcrt.py +++ b/Lib/test/test_msvcrt.py @@ -61,6 +61,19 @@ def test_get_osfhandle(self): c_encoded = b'\x57\x5b' # utf-16-le (which windows internally used) encoded char for this CJK char +def has_console(): + # A process created without a console (for example by pythonw.exe, or with + # the DETACHED_PROCESS creation flag) cannot write to the console. + try: + with open('CONOUT$', 'w'): + return True + except OSError: + return False + + +requires_console = unittest.skipUnless(has_console(), 'requires a console') + + class TestConsoleIO(unittest.TestCase): # CREATE_NEW_CONSOLE creates a "popup" window. @requires_resource('gui') @@ -106,12 +119,36 @@ def test_getche(self): def test_getwche(self): self.check_getwch('getwche') + @requires_console def test_putch(self): msvcrt.putch(b'c') + @requires_console def test_putwch(self): msvcrt.putwch(c) + def test_putch_without_console(self): + # gh-69573: putch() and putwch() must report the error instead of + # silently ignoring it when the process has no console attached. + code = dedent(''' + import msvcrt + import sys + + for name, arg in (('putch', b'c'), ('putwch', 'c')): + func = getattr(msvcrt, name) + try: + func(arg) + except OSError: + pass + else: + sys.exit(f'msvcrt.{name}() did not raise OSError') + ''') + # DETACHED_PROCESS: the child process is created without a console. + proc = subprocess.run([sys.executable, '-c', code], + creationflags=subprocess.DETACHED_PROCESS, + capture_output=True, text=True) + self.assertEqual(proc.returncode, 0, proc.stderr) + class TestOther(unittest.TestCase): def test_heap_min(self): diff --git a/Lib/test/test_multiprocessing_forkserver/__init__.py b/Lib/test/test_multiprocessing_forkserver/__init__.py index c58375e2861c62e..600285cbf613003 100644 --- a/Lib/test/test_multiprocessing_forkserver/__init__.py +++ b/Lib/test/test_multiprocessing_forkserver/__init__.py @@ -1,3 +1,4 @@ +import multiprocessing import os.path import sys import unittest @@ -12,5 +13,10 @@ if not support.has_fork_support: raise unittest.SkipTest("requires working os.fork()") +# The forkserver start method requires passing file descriptors over a Unix +# socket, which is not available on every platform (e.g. Solaris/illumos). +if "forkserver" not in multiprocessing.get_all_start_methods(): + raise unittest.SkipTest("forkserver start method is not available") + def load_tests(*args): return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index a3728b58335e63b..936332bf94ffe77 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -31,19 +31,29 @@ HAVE_GETFINALPATHNAME = True try: - import ctypes + import ctypes.util + import ctypes.wintypes except ImportError: HAVE_GETSHORTPATHNAME = False else: HAVE_GETSHORTPATHNAME = True def _getshortpathname(path): - GSPN = ctypes.WinDLL("kernel32", use_last_error=True).GetShortPathNameW - GSPN.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_uint32] - GSPN.restype = ctypes.c_uint32 + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + + @ctypes.util.wrap_dll_function(kernel32) + def GetShortPathNameW( + lpszLongPath: ctypes.c_wchar_p, + lpszShortPath: ctypes.c_wchar_p, + cchBuffer: ctypes.wintypes.DWORD, + ) -> ctypes.wintypes.DWORD: + pass + GSPN = GetShortPathNameW + result_len = GSPN(path, None, 0) if not result_len: raise OSError("failed to get short path name 0x{:08X}" .format(ctypes.get_last_error())) + result = ctypes.create_unicode_buffer(result_len) result_len = GSPN(path, result, result_len) return result[:result_len] diff --git a/Lib/test/test_os/test_os.py b/Lib/test/test_os/test_os.py index 5c258ff89ba735c..3e5ad52c4ab130d 100644 --- a/Lib/test/test_os/test_os.py +++ b/Lib/test/test_os/test_os.py @@ -156,7 +156,8 @@ def test_getcwd_long_path(self): # ("The filename or extension is too long") break except OSError as exc: - if exc.errno == errno.ENAMETOOLONG: + # DragonFly BSD raises EFAULT for a too long path. + if exc.errno in (errno.ENAMETOOLONG, errno.EFAULT): break else: raise @@ -1058,11 +1059,18 @@ def support_subsecond(self, filename): or (st.st_mtime != st[8]) or (st.st_ctime != st[9])) + def support_atime(self, filename): + # Heuristic to check if the filesystem stores the access time. + # Use whole seconds, to not depend on the timestamp resolution. + os.utime(filename, (1, 2)) + return os.stat(filename).st_atime == 1 + def _test_utime(self, set_time, filename=None): if not filename: filename = self.fname support_subsecond = self.support_subsecond(filename) + support_atime = self.support_atime(filename) if support_subsecond: # Timestamp with a resolution of 1 microsecond (10^-6). # @@ -1089,18 +1097,22 @@ def _test_utime(self, set_time, filename=None): # digits worth of sub-second precision. # Some day it would be good to fix this upstream. delta=1e-5 - self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-5) + if support_atime: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_atime_ns, atime_ns, delta=1e9 * 1e-5) self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-5) - self.assertAlmostEqual(st.st_atime_ns, atime_ns, delta=1e9 * 1e-5) self.assertAlmostEqual(st.st_mtime_ns, mtime_ns, delta=1e9 * 1e-5) else: if support_subsecond: - self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) + if support_atime: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) else: - self.assertEqual(st.st_atime, atime_ns * 1e-9) + if support_atime: + self.assertEqual(st.st_atime, atime_ns * 1e-9) self.assertEqual(st.st_mtime, mtime_ns * 1e-9) - self.assertEqual(st.st_atime_ns, atime_ns) + if support_atime: + self.assertEqual(st.st_atime_ns, atime_ns) self.assertEqual(st.st_mtime_ns, mtime_ns) def test_utime(self): @@ -4727,6 +4739,28 @@ def test_posix_pty_functions(self): son_path = os.ptsname(mother_fd) son_fd = os.open(son_path, os.O_RDWR|os.O_NOCTTY) self.addCleanup(os.close, son_fd) + if sys.platform.startswith('sunos'): + import fcntl + I_PUSH = 0x5302 + TIOCNOTTY = 0x7471 + # Pushing "ptem" makes the slave a terminal, which a session + # leader without a controlling terminal then acquires as one + # despite O_NOCTTY. Note whether we already had one. + try: + os.close(os.open('/dev/tty', os.O_RDONLY|os.O_NOCTTY)) + had_ctty = True + except OSError: + had_ctty = False + fcntl.ioctl(son_fd, I_PUSH, b'ptem\0') + fcntl.ioctl(son_fd, I_PUSH, b'ldterm\0') + if not had_ctty and os.getsid(0) == os.getpid(): + # Disown it, otherwise closing the file descriptors sends + # SIGHUP to the session. TIOCNOTTY sends it too. + old_handler = signal.signal(signal.SIGHUP, signal.SIG_IGN) + try: + fcntl.ioctl(son_fd, TIOCNOTTY) + finally: + signal.signal(signal.SIGHUP, old_handler) self.assertEqual(os.ptsname(mother_fd), os.ttyname(son_fd)) @warnings_helper.ignore_fork_in_thread_deprecation_warnings() diff --git a/Lib/test/test_os/test_posix.py b/Lib/test/test_os/test_posix.py index 8e83fa21dae6e22..41a730708974c25 100644 --- a/Lib/test/test_os/test_posix.py +++ b/Lib/test/test_os/test_posix.py @@ -46,7 +46,8 @@ def _supports_sched(): try: posix.sched_getscheduler(0) except OSError as e: - if e.errno == errno.ENOSYS: + # DragonFly BSD requires privileges to use the scheduler API. + if e.errno in (errno.ENOSYS, errno.EPERM): return False return True @@ -415,8 +416,10 @@ def test_posix_fallocate(self): if inst.errno == errno.EINVAL and sys.platform.startswith( ('sunos', 'freebsd', 'openbsd', 'gnukfreebsd')): raise unittest.SkipTest("test may fail on ZFS filesystems") - elif inst.errno == errno.EOPNOTSUPP and sys.platform.startswith("netbsd"): - raise unittest.SkipTest("test may fail on FFS filesystems") + elif inst.errno == errno.EOPNOTSUPP: + # ZFS on FreeBSD, FFS on NetBSD, etc. + raise unittest.SkipTest( + "the file system does not support posix_fallocate()") else: raise finally: @@ -821,8 +824,7 @@ def test_makedev(self): # a special case for NODEV, on others this is just an implementation # artifact. if (hasattr(posix, 'NODEV') and - sys.platform.startswith(('linux', 'macos', 'freebsd', 'dragonfly', - 'sunos'))): + sys.platform.startswith(('linux', 'macos', 'freebsd', 'sunos'))): NODEV = posix.NODEV self.assertEqual(posix.major(NODEV), NODEV) self.assertEqual(posix.minor(NODEV), NODEV) @@ -1451,6 +1453,7 @@ def test_bug_140634(self): del sched_priority, param # should not crash support.gc_collect() # just to be sure + @requires_sched @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function") def test_sched_rr_get_interval(self): try: diff --git a/Lib/test/test_os/test_windows.py b/Lib/test/test_os/test_windows.py index f1c6283f60d35ed..b21dd8a4dca6609 100644 --- a/Lib/test/test_os/test_windows.py +++ b/Lib/test/test_os/test_windows.py @@ -27,7 +27,7 @@ def _kill(self, sig): # subprocess to the parent that the interpreter is ready. When it # becomes ready, send *sig* via os.kill to the subprocess and check # that the return code is equal to *sig*. - import ctypes + import ctypes.util from ctypes import wintypes import msvcrt @@ -35,14 +35,17 @@ def _kill(self, sig): # process has exited, use PeekNamedPipe to see what's inside stdout # without waiting. This is done so we can tell that the interpreter # is started and running at a point where it could handle a signal. - PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe - PeekNamedPipe.restype = wintypes.BOOL - PeekNamedPipe.argtypes = (wintypes.HANDLE, # Pipe handle - ctypes.POINTER(ctypes.c_char), # stdout buf - wintypes.DWORD, # Buffer size - ctypes.POINTER(wintypes.DWORD), # bytes read - ctypes.POINTER(wintypes.DWORD), # bytes avail - ctypes.POINTER(wintypes.DWORD)) # bytes left + @ctypes.util.wrap_dll_function(ctypes.windll.kernel32) + def PeekNamedPipe( + hNamedPipe: wintypes.HANDLE, + lpBuffer: ctypes.POINTER(ctypes.c_char), + nBufferSize: wintypes.DWORD, + lpBytesRead: ctypes.POINTER(wintypes.DWORD), + lpTotalBytesAvail: ctypes.POINTER(wintypes.DWORD), + lpBytesLeftThisMessage: ctypes.POINTER(wintypes.DWORD), + ) -> wintypes.BOOL: + pass + msg = "running" proc = subprocess.Popen([sys.executable, "-c", "import sys;" @@ -126,10 +129,11 @@ def test_CTRL_C_EVENT(self): # Make a NULL value by creating a pointer with no argument. NULL = ctypes.POINTER(ctypes.c_int)() - SetConsoleCtrlHandler = ctypes.windll.kernel32.SetConsoleCtrlHandler - SetConsoleCtrlHandler.argtypes = (ctypes.POINTER(ctypes.c_int), - wintypes.BOOL) - SetConsoleCtrlHandler.restype = wintypes.BOOL + + @ctypes.util.wrap_dll_function(ctypes.windll.kernel32) + def SetConsoleCtrlHandler(HandlerRoutine: ctypes.POINTER(ctypes.c_int), + Add: wintypes.BOOL) -> wintypes.BOOL: + pass # Calling this with NULL and FALSE causes the calling process to # handle Ctrl+C, rather than ignore it. This property is inherited @@ -458,17 +462,20 @@ def test_getfinalpathname_handles(self): import ctypes.wintypes # noqa: F811 kernel = ctypes.WinDLL('Kernel32.dll', use_last_error=True) - kernel.GetCurrentProcess.restype = ctypes.wintypes.HANDLE + @ctypes.util.wrap_dll_function(kernel) + def GetCurrentProcess() -> ctypes.wintypes.HANDLE: + pass - kernel.GetProcessHandleCount.restype = ctypes.wintypes.BOOL - kernel.GetProcessHandleCount.argtypes = (ctypes.wintypes.HANDLE, - ctypes.wintypes.LPDWORD) + @ctypes.util.wrap_dll_function(kernel) + def GetProcessHandleCount(khProcess: ctypes.wintypes.HANDLE, + pdwHandleCount: ctypes.wintypes.LPDWORD) -> ctypes.wintypes.BOOL: + pass # This is a pseudo-handle that doesn't need to be closed - hproc = kernel.GetCurrentProcess() + hproc = GetCurrentProcess() handle_count = ctypes.wintypes.DWORD() - ok = kernel.GetProcessHandleCount(hproc, ctypes.byref(handle_count)) + ok = GetProcessHandleCount(hproc, ctypes.byref(handle_count)) self.assertEqual(1, ok) before_count = handle_count.value diff --git a/Lib/test/test_pathlib/test_join_windows.py b/Lib/test/test_pathlib/test_join_windows.py index f30c80605f7f910..5afefa61b585f3b 100644 --- a/Lib/test/test_pathlib/test_join_windows.py +++ b/Lib/test/test_pathlib/test_join_windows.py @@ -87,6 +87,11 @@ def test_vfspath(self): p = self.cls(r'\\a\b\c\d') self.assertEqual(vfspath(p), '\\\\a\\b\\c\\d') + def test_invalid_vspath(self): + msg = "expected JoinablePath object, not NoneType" + with self.assertRaisesRegex(TypeError, msg): + vfspath(None) + def test_parts(self): P = self.cls p = P(r'c:a\b') diff --git a/Lib/test/test_pathlib/test_read.py b/Lib/test/test_pathlib/test_read.py index 16fb555b2aee051..51eede9d9ebafb8 100644 --- a/Lib/test/test_pathlib/test_read.py +++ b/Lib/test/test_pathlib/test_read.py @@ -329,6 +329,11 @@ def test_info_is_symlink(self): self.assertFalse((p / 'fileA\udfff').info.is_symlink()) self.assertFalse((p / 'fileA\x00').info.is_symlink()) + def test_invalid_mode(self): + p = self.root / 'fileA' + with self.assertRaisesRegex(ValueError, 'invalid mode'): + vfsopen(p, 'q') + class ZipPathReadTest(ReadTestBase, unittest.TestCase): ground = ZipPathGround(ReadableZipPath) diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index e3aaea84ea7ce84..0825d9d980c5e7f 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2835,14 +2835,6 @@ def test_patma_264(self): self.assertEqual(y, 0) def test_patma_265(self): - x = 0.25 - 1.75j - match x: - case 0.25 - +1.75j: - y = 0 - self.assertEqual(x, 0.25 - 1.75j) - self.assertEqual(y, 0) - - def test_patma_266(self): x = 0 match x: case +1e1000: @@ -3329,6 +3321,34 @@ def test_mapping_pattern_duplicate_key_edge_case3(self): pass """) + def test_duplicate_sign_in_complex_1(self): + self.assert_syntax_error(""" + match ...: + case 0 ++ 0j: + pass + """) + + def test_duplicate_sign_in_complex_2(self): + self.assert_syntax_error(""" + match ...: + case 0 -+ 0j: + pass + """) + + def test_duplicate_sign_in_complex_3(self): + self.assert_syntax_error(""" + match ...: + case 0 +- 0j: + pass + """) + + def test_duplicate_sign_in_complex_4(self): + self.assert_syntax_error(""" + match ...: + case 0 -- 0j: + pass + """) + class TestTypeErrors(unittest.TestCase): def test_accepts_positional_subpatterns_0(self): diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 0cf5dd2e4c10042..e7b66031a86a6ca 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -5167,8 +5167,9 @@ def f(): self.assertIn('I love Python', output) - @unittest.skipIf(sys.platform.startswith('freebsd'), - '\\x08 is not interpreted as backspace on FreeBSD') + @unittest.skipIf(sys.platform.startswith(('freebsd', 'dragonfly', 'sunos')), + '\\x08 does not remove the auto-indentation with ' + 'GNU readline on this platform') def test_multiline_auto_indent(self): script = textwrap.dedent(""" import pdb; pdb.Pdb().set_trace() @@ -5207,8 +5208,9 @@ def test_multiline_completion(self): self.assertIn('42', output) - @unittest.skipIf(sys.platform.startswith('freebsd'), - '\\x08 is not interpreted as backspace on FreeBSD') + @unittest.skipIf(sys.platform.startswith(('freebsd', 'dragonfly', 'sunos')), + '\\x08 does not remove the auto-indentation with ' + 'GNU readline on this platform') def test_multiline_indent_completion(self): script = textwrap.dedent(""" import pdb; pdb.Pdb().set_trace() diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 3500f229b1b3863..dc887693840a007 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -100,14 +100,17 @@ def setUpClass(cls): with contextlib.ExitStack() as stack: python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools("venv")) - platlib_path = subprocess.check_output( - [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"], - text=True, - ).strip() - purelib_path = subprocess.check_output( - [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], - text=True, - ).strip() + + def get_sysconfig_path(name): + # Force UTF-8 to emit the non-ASCII venv path in any locale. + return subprocess.check_output( + [python_exe, "-X", "utf8", "-c", + f"import sysconfig; print(sysconfig.get_path({name!r}))"], + encoding="utf-8", + ).strip() + + platlib_path = get_sysconfig_path("platlib") + purelib_path = get_sysconfig_path("purelib") stack.enter_context(import_helper.DirsOnSysPath(platlib_path, purelib_path)) cls.addClassCleanup(stack.pop_all().close) diff --git a/Lib/test/test_profiling/test_tracing_profiler.py b/Lib/test/test_profiling/test_tracing_profiler.py index 8a7070142b60eec..6a4d968f12ef156 100644 --- a/Lib/test/test_profiling/test_tracing_profiler.py +++ b/Lib/test/test_profiling/test_tracing_profiler.py @@ -1,4 +1,5 @@ """Test suite for the cProfile module.""" +import multiprocessing import sys import unittest @@ -220,8 +221,8 @@ def test_process_spawn_pickle(self): # gh-140729: test use Process in cProfile. self._test_process_run_pickle('spawn') - @unittest.skipIf(sys.platform == 'win32', - "No 'forkserver' start method on Windows") + @unittest.skipUnless("forkserver" in multiprocessing.get_all_start_methods(), + "forkserver start method is not available") def test_process_forkserver_pickle(self): # gh-140729: test use Process in cProfile. self._test_process_run_pickle('forkserver') diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 25f94c4c740e1ea..34d30f54e9c9c7f 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -369,7 +369,7 @@ def html2text(html): Tailored for pydoc tests only. """ - html = html.replace("
    ", "\n") + html = re.sub(r"]*>", "\n", html) html = html.replace("
    ", "-"*70) html = re.sub("<.*?>", "", html) html = pydoc.replace(html, " ", " ", ">", ">", "<", "<") @@ -1913,7 +1913,7 @@ async def coro_function(ign) -> int: html = pydoc.HTMLDoc().document(coro_function) self.assertIn( - 'async coro_function', + 'async coro_function', html) def test_async_generator_annotation(self): @@ -1925,7 +1925,7 @@ async def an_async_generator(): html = pydoc.HTMLDoc().document(an_async_generator) self.assertIn( - 'async an_async_generator', + 'async an_async_generator', html) @requires_docstrings @@ -2091,7 +2091,7 @@ def test_html_doc_routines_in_module(self): doc = pydoc.HTMLDoc() result = doc.docmodule(pydocfodder) result = html2text(result) - lines = self.getsection(result, ' Functions', None) + lines = self.getsection(result, 'Functions', None) # function alias self.assertIn(' global_func_alias = global_func(x, y)', lines) self.assertIn(' A_staticmethod(x, y)', lines) diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 98be98a25278024..a2ef67baadd4afb 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -904,7 +904,7 @@ def test_trigger_leak(self): parser.ElementDeclHandler = lambda _1, _2: None self.assertRaises(TypeError, parser.Parse, data, True) - @support.skip_if_unlimited_stack_size + @support.skip_if_huge_c_stack(800_000) @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_deeply_nested_content_model(self): diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 4240a3c31749599..04a7a1b7f56751e 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -13,7 +13,7 @@ import tempfile from functools import partial from pkgutil import ModuleInfo -from unittest import TestCase, skipUnless, skipIf, SkipTest +from unittest import TestCase, skipUnless, SkipTest from unittest.mock import Mock, patch import warnings from test.support import ( @@ -37,6 +37,7 @@ code_to_events, ) from _colorize import ANSIColors, get_theme +from _pyrepl import terminfo from _pyrepl.console import Event from _pyrepl.completing_reader import stripcolor from _pyrepl._module_completer import ( @@ -1998,8 +1999,20 @@ def test_dumb_terminal_exits_cleanly(self): self.assertNotIn("Traceback", output) +def supports_pyrepl(): + # pyrepl falls back to the basic REPL if the terminal lacks any of the + # capabilities which UnixConsole requires. This covers an unset or + # "dumb" TERM as well, they are resolved to the "dumb" capabilities. + try: + info = terminfo.TermInfo(None) + except Exception: + return False + return all(info.get(cap) is not None + for cap in ("bel", "clear", "cup", "el")) + + @skipUnless(pty, "requires pty") -@skipIf((os.environ.get("TERM") or "dumb") == "dumb", "can't use pyrepl in dumb terminal") +@skipUnless(supports_pyrepl(), "can't use pyrepl in this terminal") class TestMain(ReplTestCase): def setUp(self): # Cleanup from PYTHON* variables to isolate from local diff --git a/Lib/test/test_pyrepl/test_utils.py b/Lib/test/test_pyrepl/test_utils.py index ebbd06213c69aff..e87d14304c82257 100644 --- a/Lib/test/test_pyrepl/test_utils.py +++ b/Lib/test/test_pyrepl/test_utils.py @@ -159,3 +159,29 @@ def test_gen_colors_keyword_highlighting(self): span_text = code[color.span.start:color.span.end + 1] actual_highlights.append((span_text, color.tag)) self.assertEqual(actual_highlights, expected_highlights) + + def test_gen_colors_command_highlighting(self): + cases = [ + # highlights bare command names (after stripping whitespaces) + ("exit", [("exit", "command")]), + ("quit", [("quit", "command")]), + ("copyright", [("copyright", "command")]), + ("help", [("help", "command")]), + ("clear", [("clear", "command")]), + (" clear ", [("clear", "command")]), + # no highlight when not the only token on the line + ("x = exit", [("=", "op"), ("exit", "builtin")]), + ("obj.exit", [(".", "op")]), + # falls through to builtin when called as function or used in expression + ("exit()", [("exit", "builtin"), ("(", "op"), (")", "op")]), + ("quit(0)", [("quit", "builtin"), ("(", "op"), ("0", "number"), (")", "op")]), + ("print(exit)", [("print", "builtin"), ("(", "op"), ("exit", "builtin"), (")", "op")]), + ] + for code, expected_highlights in cases: + with self.subTest(code=code): + colors = list(gen_colors(code)) + actual_highlights = [] + for color in colors: + span_text = code[color.span.start:color.span.end + 1] + actual_highlights.append((span_text, color.tag)) + self.assertEqual(actual_highlights, expected_highlights) diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index dbd3b855f536a0d..8d093ab1b7014a4 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -1082,6 +1082,14 @@ def test_binomialvariate_log_zero(self): self.assertIsInstance(result, int) self.assertIn(result, range(11)) + def test_binomialvariate_btrs_random_zero(self): + for p, expected in ((0.25, 25), (0.75, 75)): + with self.subTest(p=p): + g = random.Random() + with unittest.mock.patch.object( + g, 'random', side_effect=(0.0, 0.5, 0.5)): + self.assertEqual(g.binomialvariate(100, p), expected) + def test_constant(self): g = random.Random() N = 100 diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index b0b9d64cfe6a5f7..11ec57a259a9abf 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -5,12 +5,10 @@ import locale import os import sys +import sysconfig import tempfile import textwrap -import threading import unittest -from test import support -from test.support import threading_helper from test.support import verbose from test.support.import_helper import import_module from test.support.os_helper import unlink, temp_dir, TESTFN @@ -413,6 +411,8 @@ def test_write_read_limited_history(self): # So, we've only tested that the read did not fail. # See TestHistoryManipulation for the full test. + @unittest.skipUnless(sysconfig.get_config_var("HAVE_RL_CHANGE_ENVIRONMENT"), + "readline can modify the environment") def test_environment_is_not_modified(self): # os.environ contains environment at the time "os" module was loaded, so # before the "readline" module is loaded. @@ -442,26 +442,5 @@ def my_hook(): self.assertIs(readline.get_pre_input_hook(), my_hook) -@unittest.skipUnless(support.Py_GIL_DISABLED, 'these tests can only possibly fail with GIL disabled') -class FreeThreadingTest(unittest.TestCase): - @threading_helper.reap_threads - @threading_helper.requires_working_threading() - def test_free_threading(self): - def completer_delims(b): - b.wait() - for _ in range(100): - readline.get_completer_delims() - readline.set_completer_delims(' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?') - readline.set_completer_delims(' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?') - readline.get_completer_delims() - - count = 40 - barrier = threading.Barrier(count) - threads = [threading.Thread(target=completer_delims, args=(barrier,)) for _ in range(count)] - - with threading_helper.start_threads(threads): - pass - - if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_remote_pdb.py b/Lib/test/test_remote_pdb.py index d26d63faa61ddb6..5b23a194098b01c 100644 --- a/Lib/test/test_remote_pdb.py +++ b/Lib/test/test_remote_pdb.py @@ -8,6 +8,7 @@ import subprocess import sys import textwrap +import threading import unittest import unittest.mock from contextlib import closing, contextmanager, redirect_stdout, redirect_stderr, ExitStack @@ -18,6 +19,11 @@ import pdb from pdb import _PdbServer, _PdbClient +try: + import pty +except ImportError: + pty = None + if not sys.is_remote_debug_enabled(): raise unittest.SkipTest('remote debugging is disabled') @@ -1090,6 +1096,45 @@ def _connect_and_get_client_file(self): return process, client_file + def _connect_and_get_client_file_via_pty(self): + """Like _connect_and_get_client_file, but run the target under a pty. + + With a real terminal on stdin/stdout, PyREPL is available *inside the + target process*, which is the condition that exercises pdb's PyREPL + input handling in the remote server. + """ + controller, worker = pty.openpty() + self.addCleanup(os.close, controller) + env = dict(os.environ, TERM="xterm-256color") + env.pop("PYTHON_BASIC_REPL", None) # don't opt out of PyREPL + process = subprocess.Popen( + [sys.executable, self.script_path], + stdin=worker, stdout=worker, stderr=worker, env=env, close_fds=True, + ) + os.close(worker) # only the child keeps the worker end open + + # Continuously drain the terminal so the child never blocks on a write. + drainer = threading.Thread( + target=self._drain_until_eof, args=(controller,), daemon=True + ) + drainer.start() + self.addCleanup(drainer.join, SHORT_TIMEOUT) + + client_sock, _ = self.server_sock.accept() + client_file = client_sock.makefile('rwb') + self.addCleanup(client_file.close) + self.addCleanup(client_sock.close) + + return process, client_file + + @staticmethod + def _drain_until_eof(fd): + try: + while os.read(fd, 1024): + pass + except OSError: + pass # controller closed, or the pty went away with the child + def _read_until_prompt(self, client_file): """Helper to read messages until a prompt is received.""" messages = [] @@ -1292,6 +1337,32 @@ def test_handle_eof(self): self.assertEqual(process.returncode, 0) self.assertEqual(stderr, "") + @unittest.skipUnless(pty, "requires pty") + def test_prompt_with_interactive_terminal(self): + """The server must send "(Pdb) " even when the target owns a terminal. + + The remote server transmits its prompt string to the client, which + displays it. When the target process has an interactive terminal, + PyREPL is enabled inside it; the base pdb machinery then blanks + ``self.prompt`` (a local PyREPL would draw the prompt itself). The + remote server has no local PyREPL -- it reads input from the socket -- + so it must keep the real prompt rather than transmit an empty one. + Regression test for gh-154467, where attaching with ``pdb -p`` to a + process running at an interactive prompt showed a blank prompt. + """ + self._create_script() + process, client_file = self._connect_and_get_client_file_via_pty() + + with kill_on_error(process): + messages = self._read_until_prompt(client_file) + # The message that ended the read is the prompt request. + self.assertEqual(messages[-1], {"prompt": "(Pdb) ", "state": "pdb"}) + + # Let the target run to completion so nothing is left attached. + self._send_command(client_file, "c") + process.wait(timeout=SHORT_TIMEOUT) + self.assertEqual(process.returncode, 0) + def test_protocol_version(self): """Test that incompatible protocol versions are properly detected.""" # Create a script using an incompatible protocol version diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index 6ea27c463f31484..97d2ae4e33c548b 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -99,6 +99,11 @@ def expected(cur): # silently converted the limit value to RLIM_INFINITY. if sys.maxsize < 2**32 <= cur <= resource.RLIM_INFINITY: return [(resource.RLIM_INFINITY, max), (cur, max)] + # Solaris silently converts limits that do not fit in a signed + # 64-bit integer to RLIM_INFINITY. + if cur >= 2**63-1: + return [(resource.RLIM_INFINITY, max), + (min(cur, resource.RLIM_INFINITY), max)] return [(min(cur, resource.RLIM_INFINITY), max)] resource.setrlimit(resource.RLIMIT_FSIZE, (2**31-5, max)) diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 2adaee81b063085..4c0cd88bbcda14f 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -1,8 +1,11 @@ import io import itertools +import os import shlex import string +import tempfile import unittest +from unittest.mock import patch from test.support import cpython_only from test.support import import_helper @@ -376,6 +379,203 @@ def testPunctuationCharsReadOnly(self): with self.assertRaises(AttributeError): shlex_instance.punctuation_chars = False + def testLinenoAfterNewLine(self): + s = shlex.shlex("line 1\nline 2") + self.assertEqual(s.lineno, 1) # before consumption + list(s) + self.assertEqual(s.lineno, 2) + + def testLinenoAfterComment(self): + """Comment handler increments lineno even without a trailing newline.""" + s = shlex.shlex("line 1 # line 2") + list(s) + self.assertEqual(s.lineno, 2) + + def testPushToken(self): + s = shlex.shlex("b c") + s.push_token("a") + self.assertListEqual(list(s), ["a", "b", "c"]) + + def testPushTokenLifo(self): + s = shlex.shlex("") + s.push_token("first") + s.push_token("last") + self.assertListEqual(list(s), ["last", "first"]) + + def testPushTokenDebug(self): + s = shlex.shlex("") + s.debug = 1 + tok = "a" + with patch("builtins.print") as mock_print: + s.push_token(tok) + mock_print.assert_called_once_with(f"shlex: pushing token {tok!r}") + + def testPushSourceString(self): + s = shlex.shlex("world") + s.push_source("hello") + self.assertListEqual(list(s), ["hello", "world"]) + + def testPushSourceStream(self): + s = shlex.shlex("world") + s.push_source(io.StringIO("hello")) + self.assertListEqual(list(s), ["hello", "world"]) + + def testPushSourceStreamDebug(self): + s = shlex.shlex("") + stream = io.StringIO("hello") + s.debug = 1 + with patch("builtins.print") as mock_print: + s.push_source(stream) + mock_print.assert_called_once_with(f"shlex: pushing to stream {stream}") + + def testPushSourceNewfile(self): + """shlex.push_source sets infile to newfile; pop_source restores the original on exhaustion.""" + original_file = "original.sh" + new_file = "new.sh" + s = shlex.shlex("b", infile=original_file) + s.debug = 1 + with patch("builtins.print") as mock_print: + s.push_source("a", newfile=new_file) + mock_print.assert_called_once_with(f"shlex: pushing to file {new_file}") + self.assertEqual(s.infile, new_file) + s.debug = 0 + list(s) + self.assertEqual(s.infile, original_file) + + def testPopSourceDebug(self): + """pop_source emits debug output when debug is set.""" + s = shlex.shlex("b") + original_stream = s.instream + s.push_source("a") + s.debug = 1 + with patch("builtins.print") as mock_print: + list(s) # exhausts pushed source and triggers pop_source internally + mock_print.assert_any_call(f"shlex: popping to {original_stream}, line 1") + + def testErrorLeaderTracksPosition(self): + infile_label = "test.sh" + s = shlex.shlex("line 1\nline 2", infile=infile_label) + list(s) + result = s.error_leader() + self.assertEqual(result, f'"{infile_label}", line 2: ') + + def testErrorLeaderOverrides(self): + s = shlex.shlex("foo", infile="original.sh") + infile_label_override = "override.sh" + lineno_override = 42 + result = s.error_leader(infile=infile_label_override, lineno=lineno_override) + self.assertEqual(result, f'"{infile_label_override}", line {lineno_override}: ') + + def testNoClosingQuotation(self): + s = shlex.shlex('"foo') + with self.assertRaisesRegex(ValueError, "No closing quotation"): + list(s) + + def testNoEscapedCharacter(self): + s = shlex.shlex("\\", posix=True) + with self.assertRaisesRegex(ValueError, "No escaped character"): + list(s) + + def testSourcehookStripsQuotes(self): + with tempfile.NamedTemporaryFile(mode="w", suffix=".sh", delete_on_close=False) as f: + f.write("hello") + f.close() + s = shlex.shlex("") + newfile, stream = s.sourcehook(f'"{f.name}"') + stream.close() + self.assertEqual(newfile, f.name) + + def testSourcehookAbsolutePath(self): + with tempfile.NamedTemporaryFile(mode="w", delete_on_close=False) as f: + f.close() + s = shlex.shlex("", infile="/some/dir/main.sh") + newfile, stream = s.sourcehook(f.name) + stream.close() + self.assertEqual(newfile, f.name) + + def testSourcehookRelativePath(self): + with tempfile.TemporaryDirectory() as d: + fpath = os.path.join(d, "included.sh") + with open(fpath, "w"): + pass + s = shlex.shlex("", infile=os.path.join(d, "main.sh")) + newfile, stream = s.sourcehook("included.sh") + stream.close() + self.assertEqual(newfile, fpath) + + def testSourceInclusion(self): + """shlex.source sets a trigger keyword: when the lexer reads a token equal + to it, the next token is consumed as a filename and passed to + sourcehook, which returns a stream to push onto the input stack. + Tokens flow from that stream first, then resume from the original. + """ + s = shlex.shlex("trigger filename remaining") + s.source = "trigger" + s.sourcehook = lambda f: (f, io.StringIO("included")) + self.assertEqual(list(s), ["included", "remaining"]) + + def testGetTokenPopsPushbackDebug(self): + s = shlex.shlex("") + s.push_token("hello") + s.debug = 1 # set after push_token to isolate the pop-token branch + with patch("builtins.print") as mock_print: + tok = s.get_token() + self.assertEqual(tok, "hello") + mock_print.assert_called_once_with("shlex: popping token 'hello'") + + def testDebugWhitespaceInWhitespaceState(self): + s = shlex.shlex(" a") + s.debug = 2 + with patch("builtins.print") as mock_print: + list(s) + mock_print.assert_any_call("shlex: I see whitespace in whitespace state") + + def testDebugWhitespaceInWordState(self): + s = shlex.shlex("a b") + s.debug = 2 + with patch("builtins.print") as mock_print: + list(s) + mock_print.assert_any_call("shlex: I see whitespace in word state") + + def testDebugPunctuationInWordState(self): + s = shlex.shlex("a(") + s.debug = 2 + with patch("builtins.print") as mock_print: + list(s) + mock_print.assert_any_call("shlex: I see punctuation in word state") + + def testDebugRawToken(self): + s = shlex.shlex("hello") + s.debug = 2 + with patch("builtins.print") as mock_print: + list(s) + mock_print.assert_any_call("shlex: raw token='hello'") + + def testDebugEOFInQuote(self): + s = shlex.shlex('"oops', posix=True) + s.debug = 2 + with patch('builtins.print') as mock_print: + with self.assertRaises(ValueError): + list(s) + msgs = [call.args[0] for call in mock_print.call_args_list] + self.assertTrue(any("EOF in quotes" in m for m in msgs)) + + def testDebugEOFInEscape(self): + s = shlex.shlex("oops\\", posix=True) + s.debug = 2 + with patch("builtins.print") as mock_print: + with self.assertRaises(ValueError): + list(s) + msgs = [call.args[0] for call in mock_print.call_args_list] + self.assertTrue(any("EOF in escape" in m for m in msgs)) + + def testDebugStateTrace(self): + s = shlex.shlex("a") + s.debug = 3 + with patch("builtins.print") as mock_print: + list(s) + mock_print.assert_any_call("shlex: in state ' ' I see character: 'a'") + @cpython_only def test_lazy_imports(self): import_helper.ensure_lazy_imports('shlex', {'collections', 're', 'os'}) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 542d97e3f886e24..7bb50f7b8aa47e2 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -55,6 +55,13 @@ VSOCKPORT = 1234 AIX = platform.system() == "AIX" SOLARIS = sys.platform.startswith("sunos") +# NetBSD, OpenBSD and DragonFly deliver the file descriptors from only one +# SCM_RIGHTS control message when several are sent in a single sendmsg(). +BSD_COMBINES_SCM_RIGHTS = sys.platform.startswith( + ("netbsd", "openbsd", "dragonfly")) +# OpenBSD and DragonFly fail recvmsg() with EMSGSIZE, instead of setting +# MSG_CTRUNC, when the ancillary data buffer is too small for a cmsghdr. +CMSG_TRUNC_RAISES_EMSGSIZE = sys.platform.startswith(("openbsd", "dragonfly")) WSL = "microsoft-standard-WSL" in platform.release() try: @@ -1295,7 +1302,8 @@ def testGetServBy(self): # protocol, at least for modern Linuxes. if ( sys.platform.startswith( - ('linux', 'android', 'freebsd', 'netbsd', 'gnukfreebsd')) + ('linux', 'android', 'freebsd', 'dragonfly', 'netbsd', + 'gnukfreebsd')) or is_apple ): # avoid the 'echo' service on this platform, as there is an @@ -1779,6 +1787,23 @@ def testGetaddrinfo(self): except socket.gaierror: pass + @unittest.skipUnless(hasattr(socket, 'AI_NUMERICSERV'), + 'needs socket.AI_NUMERICSERV') + @support.thread_unsafe('setlocale is not thread-safe') + @support.run_with_locales('LC_ALL', + 'uk_UA.KOI8-U', 'uk_UA', 'ja_JP.eucJP', 'ja_JP.SJIS', 'ja_JP', + 'ko_KR.eucKR', 'zh_CN.GB18030', 'el_GR.ISO8859-7', + 'de_DE.ISO8859-1', 'ja_JP.UTF-8', + '') + def test_getaddrinfo_localized_error(self): + # gh-93251: the localized gai_strerror() message could fail to be + # decoded as UTF-8, so UnicodeDecodeError was raised + # instead of gaierror. + with self.assertRaises(socket.gaierror) as cm: + socket.getaddrinfo("localhost", "http", + flags=socket.AI_NUMERICSERV) + str(cm.exception) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_getaddrinfo_int_port_overflow(self): # gh-74895: Test that getaddrinfo does not raise OverflowError on port. @@ -4178,6 +4203,7 @@ def _testFDPassCMSG_LEN(self): @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") + @unittest.skipIf(BSD_COMBINES_SCM_RIGHTS, "skipping, see gh-125860") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparate(self): # Pass two FDs in two separate arrays. Arrays may be combined @@ -4190,6 +4216,7 @@ def testFDPassSeparate(self): @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") + @unittest.skipIf(BSD_COMBINES_SCM_RIGHTS, "skipping, see gh-125860") def _testFDPassSeparate(self): fd0, fd1 = self.newFDs(2) self.assertEqual( @@ -4204,6 +4231,7 @@ def _testFDPassSeparate(self): @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") + @unittest.skipIf(BSD_COMBINES_SCM_RIGHTS, "skipping, see gh-125860") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparateMinSpace(self): # Pass two FDs in two separate arrays, receiving them into the @@ -4219,6 +4247,7 @@ def testFDPassSeparateMinSpace(self): @unittest.skipIf(is_apple, "skipping, see issue #12958") @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") + @unittest.skipIf(BSD_COMBINES_SCM_RIGHTS, "skipping, see gh-125860") def _testFDPassSeparateMinSpace(self): fd0, fd1 = self.newFDs(2) self.assertEqual( @@ -4343,6 +4372,7 @@ def _testCmsgTrunc0(self): # (but still too small) buffer sizes. @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTrunc1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) @@ -4351,6 +4381,7 @@ def _testCmsgTrunc1(self): self.createAndSendFDs(1) @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTrunc2Int(self): # The cmsghdr structure has at least three members, two of # which are ints, so we still shouldn't see any ancillary @@ -4363,6 +4394,7 @@ def _testCmsgTrunc2Int(self): self.createAndSendFDs(1) @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTruncLen0Minus1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), socket.CMSG_LEN(0) - 1)) @@ -4400,6 +4432,7 @@ def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): self.checkFDs(fds) @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTruncLen0(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) @@ -4408,6 +4441,7 @@ def _testCmsgTruncLen0(self): self.createAndSendFDs(1) @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTruncLen0Plus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) @@ -4416,6 +4450,7 @@ def _testCmsgTruncLen0Plus1(self): self.createAndSendFDs(2) @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTruncLen1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), maxdata=SIZEOF_INT) @@ -4426,6 +4461,7 @@ def _testCmsgTruncLen1(self): @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTruncLen2Minus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, maxdata=(2 * SIZEOF_INT) - 1) @@ -4698,6 +4734,7 @@ def _testSingleCmsgTrunc0(self): # (but still too small) buffer sizes. @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSingleCmsgTrunc1(self): self.checkHopLimitTruncatedHeader(ancbufsize=1) @@ -4707,6 +4744,7 @@ def _testSingleCmsgTrunc1(self): self.sendToServer(MSG) @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSingleCmsgTrunc2Int(self): self.checkHopLimitTruncatedHeader(ancbufsize=2 * SIZEOF_INT) @@ -4716,6 +4754,7 @@ def _testSingleCmsgTrunc2Int(self): self.sendToServer(MSG) @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSingleCmsgTruncLen0Minus1(self): self.checkHopLimitTruncatedHeader(ancbufsize=socket.CMSG_LEN(0) - 1) @@ -4725,6 +4764,7 @@ def _testSingleCmsgTruncLen0Minus1(self): self.sendToServer(MSG) @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSingleCmsgTruncInData(self): # Test truncation of a control message inside its associated # data. The message may be returned with its data truncated, @@ -4797,6 +4837,7 @@ def _testSecondCmsgTrunc0(self): @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", "IPV6_RECVTCLASS", "IPV6_TCLASS") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSecondCmsgTrunc1(self): self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 1) @@ -4807,6 +4848,7 @@ def _testSecondCmsgTrunc1(self): @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", "IPV6_RECVTCLASS", "IPV6_TCLASS") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSecondCmsgTrunc2Int(self): self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 2 * SIZEOF_INT) @@ -4818,6 +4860,7 @@ def _testSecondCmsgTrunc2Int(self): @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", "IPV6_RECVTCLASS", "IPV6_TCLASS") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSecondCmsgTruncLen0Minus1(self): self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(0) - 1) @@ -4829,6 +4872,7 @@ def _testSecondCmsgTruncLen0Minus1(self): @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", "IPV6_RECVTCLASS", "IPV6_TCLASS") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSecondCmsgTruncInData(self): # Test truncation of the second of two control messages inside # its associated data. diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index 1fc0236780fa8ba..0d235d90f1101b4 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -223,6 +223,9 @@ def setUpClass(cls): readline = import_module("readline") if readline.backend == "editline": raise unittest.SkipTest("libedit readline is not supported") + if sys.platform.startswith("openbsd"): + # OpenBSD's readline does not honor "completion-query-items 0". + raise unittest.SkipTest("OpenBSD readline hangs on completion") def write_input(self, input_, env=None): script = textwrap.dedent(""" @@ -378,10 +381,10 @@ def test_complete_schemata(self): self.assertIn("main", candidates) self.assertIn("temp", candidates) - @unittest.skipIf(sys.platform.startswith("freebsd"), + @unittest.skipIf(sys.platform.startswith(("freebsd", "dragonfly", "sunos")), "Two actual tabs are inserted when there are no matching" " completions in the pseudo-terminal opened by run_pty()" - " on FreeBSD") + " on this platform") def test_complete_no_match(self): input_ = b"xyzzy\t\t\b\b\b\b\b\b\b.quit\n" # Set NO_COLOR to disable coloring for self.PS1. diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 352a9c2a0c6106f..6446f96eab42a43 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -49,12 +49,15 @@ PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) HOST = socket_helper.HOST IS_AWS_LC = "AWS-LC" in ssl.OPENSSL_VERSION +IS_LIBRESSL = "LibreSSL" in ssl.OPENSSL_VERSION IS_OPENSSL_3_0_0 = ssl.OPENSSL_VERSION_INFO >= (3, 0, 0) CAN_GET_SELECTED_OPENSSL_GROUP = ssl.OPENSSL_VERSION_INFO >= (3, 2) CAN_IGNORE_UNKNOWN_OPENSSL_GROUPS = ssl.OPENSSL_VERSION_INFO >= (3, 3) CAN_GET_AVAILABLE_OPENSSL_GROUPS = ssl.OPENSSL_VERSION_INFO >= (3, 5) CAN_GET_AVAILABLE_OPENSSL_SIGALGS = ssl.OPENSSL_VERSION_INFO >= (3, 4) -CAN_SET_CLIENT_SIGALGS = not IS_AWS_LC +# LibreSSL does not provide SSL_CTX_set1_(client_)sigalgs_list(). +CAN_SET_CLIENT_SIGALGS = not IS_AWS_LC and not IS_LIBRESSL +CAN_SET_SERVER_SIGALGS = not IS_LIBRESSL CAN_IGNORE_UNKNOWN_OPENSSL_SIGALGS = ssl.OPENSSL_VERSION_INFO >= (3, 3) CAN_GET_SELECTED_OPENSSL_SIGALG = ssl.OPENSSL_VERSION_INFO >= (3, 5) PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') @@ -1080,6 +1083,8 @@ def test_set_client_sigalgs(self): if CAN_IGNORE_UNKNOWN_OPENSSL_SIGALGS: self.assertIsNone(ctx.set_client_sigalgs('rsa_pss_rsae_sha256:?foo')) + @unittest.skipUnless(CAN_SET_SERVER_SIGALGS, + "SSL library doesn't support setting server sigalgs") def test_set_server_sigalgs(self): ctx = ssl.create_default_context() @@ -4593,6 +4598,8 @@ def test_client_sigalgs_mismatch(self): chatty=True, connectionchatty=True, sni_name=hostname) + @unittest.skipUnless(CAN_SET_SERVER_SIGALGS, + "SSL library doesn't support setting server sigalgs") def test_server_sigalgs(self): # server rsa_pss_rsae_sha384, client auto sigalg = "rsa_pss_rsae_sha384" @@ -4613,6 +4620,8 @@ def test_server_sigalgs(self): if CAN_GET_SELECTED_OPENSSL_SIGALG: self.assertEqual(stats['server_sigalg'], sigalg) + @unittest.skipUnless(CAN_SET_SERVER_SIGALGS, + "SSL library doesn't support setting server sigalgs") def test_server_sigalgs_mismatch(self): client_context, server_context, hostname = testing_context() client_context.set_server_sigalgs("rsa_pss_rsae_sha256") diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index d556f96bc532ed1..4b9bc245d6f78a8 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -19,6 +19,7 @@ import warnings from test import support +from test.support import isolation from test.support import hashlib_helper from test.support import import_helper from test.support import os_helper @@ -1075,5 +1076,125 @@ def test_disable_hash_md5_in_fips_mode_allow_all(self): self.assertIsInstance(h, self._hashlib.HASH) +class TestIsolated(unittest.TestCase): + # Drive the sample tests in test._isolated_sample (which really spawn + # subprocesses through @isolation.runInSubprocess()) under a private + # TestResult, and check that each subprocess outcome is replayed in the parent. + + @staticmethod + def _run(name): + suite = unittest.TestLoader().loadTestsFromName( + 'test._isolated_sample.' + name) + result = unittest.TestResult() + suite.run(result) + return result + + @staticmethod + def _names(items): + # Map outcome entries (which are (test, detail) pairs, except + # unexpectedSuccesses which are bare tests) to their method names. + names = [] + for item in items: + test = item[0] if isinstance(item, tuple) else item + names.append(test.id().rpartition('.')[2]) + return sorted(names) + + @support.requires_subprocess() + def test_method_outcomes(self): + result = self._run('MethodSample') + self.assertEqual(result.testsRun, 6) + self.assertEqual(self._names(result.failures), ['test_fail']) + self.assertEqual(self._names(result.errors), ['test_error']) + self.assertEqual(self._names(result.skipped), ['test_skip']) + self.assertEqual(self._names(result.expectedFailures), + ['test_expected_failure']) + self.assertEqual(self._names(result.unexpectedSuccesses), + ['test_unexpected_success']) + + @support.requires_subprocess() + def test_class_outcomes(self): + result = self._run('ClassSample') + self.assertEqual(result.testsRun, 3) + self.assertEqual(self._names(result.failures), ['test_fail']) + self.assertEqual(self._names(result.expectedFailures), + ['test_expected_failure']) + self.assertEqual(result.errors, []) + self.assertEqual(result.unexpectedSuccesses, []) + + @support.requires_subprocess() + def test_subtests_reported_individually(self): + result = self._run('SubtestSample') + self.assertEqual(result.testsRun, 1) + self.assertEqual(len(result.failures), 1) + test, _ = result.failures[0] + self.assertIn('i=1', str(test)) + + @support.requires_subprocess() + def test_skip_reason_propagated(self): + result = self._run('MethodSample.test_skip') + self.assertEqual([reason for _, reason in result.skipped], ['nope']) + + @support.requires_subprocess() + def test_subprocess_traceback_is_cause(self): + result = self._run('MethodSample.test_fail') + self.assertEqual(len(result.failures), 1) + _, tb = result.failures[0] + # The real assertion that failed in the subprocess is shown ... + self.assertIn('self.assertEqual(1, 2)', tb) + # ... as the direct cause of the replayed failure ... + self.assertIn('direct cause', tb) + # ... without leaking the parent-side replay frames. + self.assertNotIn('isolation.py', tb) + + @support.requires_subprocess() + def test_durations_forwarded_for_class(self): + from test._isolated_sample import DURATION_SLEEP + result = unittest.TestResult() + suite = unittest.TestLoader().loadTestsFromName( + 'test._isolated_sample.DurationSample') + suite.run(result) + # The duration reported in the parent is the one measured in the + # subprocess (around the sleep), not the near-instant replay time. + self.assertEqual(len(result.collectedDurations), 1) + name, elapsed = result.collectedDurations[0] + self.assertEqual(name.split()[0], 'test_slow') + self.assertGreaterEqual(elapsed, DURATION_SLEEP / 2) + + @support.requires_subprocess() + def test_subclass_of_isolated_class(self): + # Both samples pass only if the fixtures are bound to the runtime class + # and what the subclass adds or overrides runs in the subprocess. + for name, count in (('SubclassingSample', 1), ('SubclassSample', 2)): + with self.subTest(sample=name): + result = self._run(name) + self.assertEqual(result.testsRun, count) + self.assertEqual(result.failures, []) + self.assertEqual(result.errors, []) + + @support.requires_subprocess() + def test_subclass_bypassing_setupclass_is_reported(self): + # A class that never ran in a subprocess must error out, not pass with + # no outcome to replay. + result = self._run('BrokenSubclassSample') + self.assertEqual(result.testsRun, 1) + self.assertEqual(len(result.errors), 1) + self.assertIn('did not run in a subprocess', result.errors[0][1]) + + def test_skipped_without_subprocess_support(self): + # On a platform without subprocess support the test is skipped in the + # parent, before any subprocess is spawned. + calls = [] + orig = isolation._run_in_subprocess + with support.swap_attr(support, 'has_subprocess_support', False): + isolation._run_in_subprocess = lambda *a, **k: calls.append(a) + try: + result = self._run('MethodSample.test_pass') + finally: + isolation._run_in_subprocess = orig + self.assertEqual(result.testsRun, 1) + self.assertEqual(len(result.skipped), 1) + self.assertEqual(calls, []) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 56c5c2aa2025b9a..dab03ef06a8b8e5 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1809,7 +1809,7 @@ def delx(self): del self.__x check((1,2,3), vsize('') + self.P + 3*self.P) # type # static type: PyTypeObject - fmt = 'P2nPI13Pl4Pn9Pn12PI2Pc' + fmt = 'P2nPI13Pl4Pn9Pn12PI2PcP' s = vsize(fmt) check(int, s) typeid = 'n' if support.Py_GIL_DISABLED else '' diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index e6f99581f0b7a66..9bb3e326ff3e952 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -457,6 +457,15 @@ def test_soabi(self): soabi = sysconfig.get_config_var('SOABI') self.assertIn(soabi, _imp.extension_suffixes()[0]) + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") + @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows") + @unittest.skipIf(sysconfig.get_config_var('SOABI_PLATFORM') == 0, + "SOABI_PLATFORM is undefined") + def test_soabi_platform(self): + soabi_platform = sysconfig.get_config_var('SOABI_PLATFORM') + soabi = sysconfig.get_config_var('SOABI') + self.assertIn(soabi_platform, soabi) + def test_library(self): library = sysconfig.get_config_var('LIBRARY') ldlibrary = sysconfig.get_config_var('LDLIBRARY') diff --git a/Lib/test/test_termios.py b/Lib/test/test_termios.py index 3dcf5bd13791dc1..2739b990e0906e5 100644 --- a/Lib/test/test_termios.py +++ b/Lib/test/test_termios.py @@ -117,7 +117,7 @@ def test_tcsetattr_errors(self): @support.skip_android_selinux('tcsendbreak') def test_tcsendbreak(self): with skip_enotty_error(self, 'tcsendbreak', - ('freebsd', 'netbsd', 'cygwin')): + ('freebsd', 'netbsd', 'openbsd', 'dragonfly', 'cygwin')): termios.tcsendbreak(self.fd, 1) termios.tcsendbreak(self.stream, 1) @@ -194,6 +194,9 @@ def test_tcflow(self): termios.tcflow(self.fd, termios.TCOON) termios.tcflow(self.fd, termios.TCIOFF) termios.tcflow(self.fd, termios.TCION) + # Discard the transmitted STOP and START characters, + # otherwise closing the pseudo-terminal can block. + termios.tcflush(self.fd, termios.TCOFLUSH) @support.skip_android_selinux('tcflow') def test_tcflow_errors(self): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 3d01804513bde98..96b43936be92cda 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -326,9 +326,13 @@ def f(mutex): @cpython_only def test_PyThreadState_SetAsyncExc(self): ctypes = import_module("ctypes") + import ctypes.util # noqa: F811 - set_async_exc = ctypes.pythonapi.PyThreadState_SetAsyncExc - set_async_exc.argtypes = (ctypes.c_ulong, ctypes.py_object) + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyThreadState_SetAsyncExc(id: ctypes.c_ulong, + exc: ctypes.py_object) -> ctypes.c_int: + pass + set_async_exc = PyThreadState_SetAsyncExc class AsyncExc(Exception): pass @@ -485,7 +489,17 @@ def test_finalize_running_thread(self): import_module("ctypes") rc, out, err = assert_python_failure("-c", """if 1: - import ctypes, sys, time, _thread + import ctypes.util, sys, time, _thread + + PyGILState_STATE = ctypes.c_int # enum + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyGILState_Ensure() -> PyGILState_STATE: + pass + + @ctypes.util.wrap_dll_function(ctypes.pythonapi) + def PyGILState_Release(oldstate: PyGILState_STATE) -> None: + pass # This lock is used as a simple event variable. ready = _thread.allocate_lock() @@ -494,8 +508,8 @@ def test_finalize_running_thread(self): # Module globals are cleared before __del__ is run # So we save the functions in class dict class C: - ensure = ctypes.pythonapi.PyGILState_Ensure - release = ctypes.pythonapi.PyGILState_Release + ensure = PyGILState_Ensure + release = PyGILState_Release def __del__(self): state = self.ensure() self.release(state) diff --git a/Lib/test/test_tools/test_compute_changes.py b/Lib/test/test_tools/test_compute_changes.py index eb1ecd447a7ce0a..4c2b9966c4cd5b6 100644 --- a/Lib/test/test_tools/test_compute_changes.py +++ b/Lib/test/test_tools/test_compute_changes.py @@ -109,15 +109,6 @@ def test_unix(self): self.assertTrue(result.run_tests) self.assertFalse(result.run_windows_tests) - def test_msi(self): - for f in ( - ".github/workflows/reusable-windows-msi.yml", - "Tools/msi/build.bat", - ): - with self.subTest(f=f): - result = process_changed_files({Path(f)}) - self.assertTrue(result.run_windows_msi) - def test_all_run(self): for f in ( ".github/workflows/some-new-workflow.yml", diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 849a2afd8ed7986..9827f2498554a5b 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -1,6 +1,7 @@ """ Tests for the internal type cache in CPython. """ import collections.abc import dis +import sys import unittest import warnings from test import support @@ -281,5 +282,163 @@ def to_bool_2(instance): self._check_specialization(to_bool_2, H(), "TO_BOOL", should_specialize=False) +@support.cpython_only +class PerTypeLookupCacheTests(unittest.TestCase): + """Tests for the per-type lookup cache.""" + + type_cache_lookup = staticmethod(_testinternalcapi.type_cache_lookup) + type_cache_invalidate = staticmethod(_testinternalcapi.type_cache_invalidate) + + def _make_type(self): + class C: + x = "x-value" + return C + + def test_lookup_miss_on_empty_cache(self): + # A freshly-created type has not cached any names yet; the cache + # should report a miss for an arbitrary name. + C = self._make_type() + hit, value, version = self.type_cache_lookup(C, "x") + self.assertEqual(hit, 0) + self.assertIsNone(value) + self.assertEqual(version, 0) + + def test_lookup_hit_after_access(self): + # Reading an attribute goes through _PyType_Lookup which + # caches the result. Subsequent lookups for the same name + # should hit the cache. + C = self._make_type() + hit, value, version = self.type_cache_lookup(C, "x") + self.assertEqual(hit, 0) + attr = C.x + hit, value, version = self.type_cache_lookup(C, "x") + self.assertEqual(hit, 1) + self.assertIs(value, attr) + self.assertNotEqual(version, 0) + self.assertEqual(version, type_get_version(C)) + + def test_lookup_caches_missing_name(self): + # _PyType_Lookup caches negative results too: a name that is not in + # the MRO should still produce a cache hit with a None value. + C = self._make_type() + with self.assertRaises(AttributeError): + C.does_not_exist + hit, value, _ = self.type_cache_lookup(C, "does_not_exist") + self.assertEqual(hit, 1) + self.assertIsNone(value) + + def test_lookup_on_static_type(self): + # The cache for static types is stored on interpreter for isolation + # between subinterpreters, test that cache works for them as well. + self.type_cache_invalidate(int) + name = sys.intern("bit_length") + self.assertEqual(self.type_cache_lookup(int, name)[0], 0) + attr = getattr(int, name) + hit, value, _ = self.type_cache_lookup(int, name) + self.assertEqual(hit, 1) + self.assertIs(value, attr) + + def test_invalidate_clears_cache(self): + C = self._make_type() + C.x # populate cache + self.assertEqual(self.type_cache_lookup(C, "x")[0], 1) + + self.type_cache_invalidate(C) + hit, value, _ = self.type_cache_lookup(C, "x") + self.assertEqual(hit, 0) + self.assertIsNone(value) + + def test_setattr_invalidates_cache(self): + # Mutating a type's attributes must invalidate any cached entries + # for that type. + C = self._make_type() + C.x + self.assertEqual(self.type_cache_lookup(C, "x")[0], 1) + + C.x = "new-value" + hit, _, _ = self.type_cache_lookup(C, "x") + self.assertEqual(hit, 0) + + # The next access should re-populate the cache with the new value. + self.assertEqual(C.x, "new-value") + hit, value, _ = self.type_cache_lookup(C, "x") + self.assertEqual(hit, 1) + self.assertEqual(value, "new-value") + + def test_setattr_on_subclass_preserves_base(self): + # Adding an attribute to a subclass changes the lookup result for + # the subclass, so its cache must be invalidated, but the base's + # cache for the same name stays valid. + class Base: + x = "base" + class Sub(Base): + pass + + self.assertEqual(Sub.x, "base") + self.assertEqual(Base.x, "base") + self.assertEqual(self.type_cache_lookup(Sub, "x")[0], 1) + self.assertEqual(self.type_cache_lookup(Base, "x")[0], 1) + + Sub.x = "sub" + # Sub's cache should be invalidated. + self.assertEqual(self.type_cache_lookup(Sub, "x")[0], 0) + # Base is untouched. + hit, value, _ = self.type_cache_lookup(Base, "x") + self.assertEqual(hit, 1) + self.assertEqual(value, "base") + + def test_setattr_on_base_invalidates_subclass(self): + class Base: + x = "base" + class Sub(Base): + pass + + Sub.x + self.assertEqual(self.type_cache_lookup(Sub, "x")[0], 1) + + Base.x = "new-base" + # Modifying the base must invalidate the subclass cache too. + self.assertEqual(self.type_cache_lookup(Sub, "x")[0], 0) + + def test_lookup_detects_stale_cache_version(self): + # The cache stores the type's tp_version_tag alongside its entries + # and re-checks it after locating a hit. If the type version moves + # forward without the cache being invalidated (the race window in + # lock-free invalidation), the consistency check must downgrade + # the hit to a miss. + C = self._make_type() + C.x # populate cache + orig_version = type_get_version(C) + self.assertNotEqual(orig_version, 0) + self.assertEqual(self.type_cache_lookup(C, "x")[0], 1) + + # Bump the type version directly without touching the cache slot + # (PyType_Modified would also invalidate, defeating the test). + type_assign_specific_version_unsafe(C, orig_version + 1) + self.assertEqual(type_get_version(C), orig_version + 1) + + hit, value, _ = self.type_cache_lookup(C, "x") + self.assertEqual(hit, 0) + self.assertIsNone(value) + + def test_setattr_on_unrelated_type_preserves_cache(self): + # Modifying one type must not invalidate a sibling's cache. + class A: + x = "a" + class B: + x = "b" + + A.x + B.x + self.assertEqual(self.type_cache_lookup(A, "x")[0], 1) + self.assertEqual(self.type_cache_lookup(B, "x")[0], 1) + + B.x = "b2" + # A's cache is unaffected. + hit, value, _ = self.type_cache_lookup(A, "x") + self.assertEqual(hit, 1) + self.assertEqual(value, "a") + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 17574c7f03d8e1e..106ffdede6fd4ef 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -52,6 +52,8 @@ from test.support import ( captured_stderr, cpython_only, requires_docstrings, import_helper, run_code, subTests, EqualToForwardRef, + exceeds_recursion_limit, skip_if_huge_c_stack, skip_wasi_stack_overflow, + skip_emscripten_stack_overflow, ) from test.typinganndata import ( ann_module695, mod_generics_cache, _typed_dict_helper, @@ -2801,8 +2803,14 @@ def test_args(self): self.assertEqual(Literal[1, 2, 3].__args__, (1, 2, 3)) self.assertEqual(Literal[1, 2, 3, 3].__args__, (1, 2, 3)) self.assertEqual(Literal[1, Literal[2], Literal[3, 4]].__args__, (1, 2, 3, 4)) - # Mutable arguments will not be deduplicated - self.assertEqual(Literal[[], []].__args__, ([], [])) + # Unhashable arguments will be deduplicated too + self.assertEqual(Literal[[], []].__args__, ([],)) + self.assertEqual(Literal[{"a": 1}, {"a": 1}].__args__, ({"a": 1},)) + self.assertEqual( + Literal[1, {'a': 'b'}, 2, {'a': 'b'}, 3].__args__, + (1, {'a': 'b'}, 2, 3), + ) + self.assertEqual(Literal[{1}, {1}, {2}, {2}].__args__, ({1}, {2})) def test_flatten(self): l1 = Literal[Literal[1], Literal[2], Literal[3]] @@ -5088,6 +5096,17 @@ class MM2(collections.abc.MutableMapping, MutableMapping[str, str]): pass self.assertEqual(MM2.__bases__, (collections.abc.MutableMapping, Generic)) + @cpython_only + @skip_if_huge_c_stack() + @skip_wasi_stack_overflow() + @skip_emscripten_stack_overflow() + def test_parameters_deep_recursion(self): + x = [0] + for _ in range(exceeds_recursion_limit()): + x = [x] + with self.assertRaisesRegex(RecursionError, "in __parameter__ calculation"): + list[x].__parameters__ + def test_orig_bases(self): T = TypeVar('T') class C(typing.Dict[str, T]): ... diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 2dd739b77b8e4d1..1e5f79998e7cab2 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -467,6 +467,25 @@ def test_redirect_limit_independent(self): finally: self.unfakehttp() + def test_http_error_attribute_values(self): + hdrs = { + "Authorization": "Bearer foobar", + "Accept": "application/json" + } + err = urllib.error.HTTPError("http://something", 404, "foo", hdrs, None) + self.assertEqual(err.filename, "http://something") + self.assertEqual(err.code, 404) + self.assertEqual(err.msg, "foo") + self.assertEqual(err.reason, "foo") + self.assertEqual(err.hdrs, hdrs) + self.assertEqual(err.headers, hdrs) + err.close() + + def test_http_error_default_fp(self): + err = urllib.error.HTTPError("http://something", 404, "foo", {}, None) + self.assertIsInstance(err.fp, io.BytesIO) + err.close() + def test_empty_socket(self): # urlopen() raises OSError if the underlying socket does not send any # data. (#1680230) @@ -513,6 +532,11 @@ def test_ftp_nonexisting(self): self.assertFalse(e.exception.filename) self.assertTrue(e.exception.reason) + def test_url_error_stringified(self): + reason = 'sixseven' + err = urllib.error.URLError(reason) + self.assertEqual(str(err), f'') + class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 58ae85fb268042e..e98e52c2ea20453 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -783,8 +783,16 @@ def test_fish_activate_shadowed_builtins(self): shadows one of those builtins (a common pattern for `.`-style directory navigators) must not hijack the prompt or break status restoration. """ - fish = shutil.which('fish') - if fish is None: + # Some systems have an unrelated "fish" game (see fish(6)), which + # can precede the fish shell in PATH. + for dirname in None, *os.get_exec_path(): + fish = shutil.which('fish', path=dirname) + if fish is not None and subprocess.run( + [fish, '-c', 'echo $FISH_VERSION'], + stdin=subprocess.DEVNULL, + capture_output=True).stdout.strip(): + break + else: self.skipTest('fish required for this test') rmtree(self.env_dir) builder = venv.EnvBuilder(clear=True) diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py index a1c0b80d47e4d44..0ae03a3bf505f73 100644 --- a/Lib/test/test_winapi.py +++ b/Lib/test/test_winapi.py @@ -12,9 +12,16 @@ MAXIMUM_WAIT_OBJECTS = 64 MAXIMUM_BATCHED_WAIT_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) ** 2 + +def close_events(events): + for handle in events: + _winapi.CloseHandle(handle) + + class WinAPIBatchedWaitForMultipleObjectsTests(unittest.TestCase): def _events_waitall_test(self, n): evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + self.addCleanup(close_events, evts) with self.assertRaises(TimeoutError): _winapi.BatchedWaitForMultipleObjects(evts, True, 100) @@ -42,6 +49,7 @@ def _events_waitall_test(self, n): def _events_waitany_test(self, n): evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + self.addCleanup(close_events, evts) with self.assertRaises(TimeoutError): _winapi.BatchedWaitForMultipleObjects(evts, False, 100) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index acec4ec2ca257c4..38bc681a267b951 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -3246,7 +3246,7 @@ def __deepcopy__(self, memo): self.assertEqual([c.tag for c in children[3:]], [a.tag, b.tag, a.tag, b.tag]) - @support.skip_if_unlimited_stack_size + @support.skip_if_huge_c_stack(500_000) @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_deeply_nested_deepcopy(self): @@ -3477,6 +3477,37 @@ def test_find_xpath(self): self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()-0]') self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()+1]') + def test_find_xpath_index_no_quadratic_complexity(self): + class CountingElement(ET.Element): + findall_calls = 0 + def findall(self, *args, **kwargs): + type(self).findall_calls += 1 + return super().findall(*args, **kwargs) + + def work(n, pattern): + root = CountingElement("root") + for _ in range(n): + ET.SubElement(root, "a") + CountingElement.findall_calls = 0 + root.findall(pattern) + return CountingElement.findall_calls + + for pattern in [".//a[1]", ".//a[last()]"]: + w1 = work(1024, pattern) + w2 = work(2048, pattern) + w3 = work(4096, pattern) + + self.assertGreater(w1, 0) + r1 = w2 / w1 + r2 = w3 / w2 + # Doubling N must not ~double the parent.findall calls. + # Linear-in-N call counts indicate the cache is missing. + self.assertLess( + max(r1, r2), 1.5, + msg=f"Possible quadratic behavior on {pattern!r}: " + f"calls={w1, w2, w3} ratios={r1, r2}", + ) + def test_findall(self): e = ET.XML(SAMPLE_XML) e[2] = ET.XML(SAMPLE_SECTION) diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 4f20209927e7b3d..cd498ba13e6f461 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1,6 +1,7 @@ import _pyio import array import contextlib +import errno import importlib.util import io import itertools @@ -655,6 +656,12 @@ def test_add_file_after_2107(self): os.utime(TESTFN, (ts, ts)) except OverflowError: self.skipTest('Host fs cannot set timestamp to required value.') + except OSError as exc: + # Some file systems (e.g. UFS and ZFS on illumos) do not + # support timestamps that do not fit in 32 bits. + if exc.errno != errno.EOVERFLOW: + raise + self.skipTest('Host fs cannot set timestamp to required value.') mtime_ns = os.stat(TESTFN).st_mtime_ns if mtime_ns != (4386268800 * 10**9): diff --git a/Lib/test/win_console_handler.py b/Lib/test/win_console_handler.py index e7779b9363503b4..a1c7c42c109330b 100644 --- a/Lib/test/win_console_handler.py +++ b/Lib/test/win_console_handler.py @@ -15,7 +15,7 @@ import sys # Function prototype for the handler function. Returns BOOL, takes a DWORD. -HandlerRoutine = WINFUNCTYPE(wintypes.BOOL, wintypes.DWORD) +HANDLER_ROUTINE = WINFUNCTYPE(wintypes.BOOL, wintypes.DWORD) def _ctrl_handler(sig): """Handle a sig event and return 0 to terminate the process""" @@ -27,12 +27,13 @@ def _ctrl_handler(sig): print("UNKNOWN EVENT") return 0 -ctrl_handler = HandlerRoutine(_ctrl_handler) +ctrl_handler = HANDLER_ROUTINE(_ctrl_handler) -SetConsoleCtrlHandler = ctypes.windll.kernel32.SetConsoleCtrlHandler -SetConsoleCtrlHandler.argtypes = (HandlerRoutine, wintypes.BOOL) -SetConsoleCtrlHandler.restype = wintypes.BOOL +@ctypes.util.wrap_dll_function(ctypes.windll.kernel32) +def SetConsoleCtrlHandler(HandlerRoutine: HANDLER_ROUTINE, + Add: wintypes.BOOL) -> wintypes.BOOL: + pass if __name__ == "__main__": # Add our console control handling function with value 1 diff --git a/Lib/typing.py b/Lib/typing.py index 933336ff4cf37e2..054420865d7fb50 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -775,13 +775,16 @@ def open_helper(file: str, mode: MODE) -> str: # There is no '_type_check' call because arguments to Literal[...] are # values, not types. parameters = _flatten_literal_params(parameters) + value_and_type_parameters = list(_value_and_type_iter(parameters)) + deduplicated_parameters = tuple( + p + for p, _ in _deduplicate( + value_and_type_parameters, + unhashable_fallback=True, + ) + ) - try: - parameters = tuple(p for p, _ in _deduplicate(list(_value_and_type_iter(parameters)))) - except TypeError: # unhashable parameters - pass - - return _LiteralGenericAlias(self, parameters) + return _LiteralGenericAlias(self, deduplicated_parameters) @_SpecialForm diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index 2a3fa835476ab9b..7226034df0f3941 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -16,10 +16,12 @@ setenv PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" setenv VIRTUAL_ENV_PROMPT __VENV_PROMPT__ -set _OLD_VIRTUAL_PROMPT="$prompt" +if ($?prompt) then + set _OLD_VIRTUAL_PROMPT="$prompt" -if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "("__VENV_PROMPT__") $prompt:q" + if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + set prompt = "("__VENV_PROMPT__") $prompt:q" + endif endif alias pydoc python -m pydoc diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py index dc6bd28c03137de..de1fd203ff6a0f6 100644 --- a/Lib/xml/etree/ElementPath.py +++ b/Lib/xml/etree/ElementPath.py @@ -324,15 +324,22 @@ def select_negated(context, result): index = -1 def select(context, result): parent_map = get_parent_map(context) + cache = {} for elem in result: try: parent = parent_map[elem] + except KeyError: + continue + key = (parent, elem.tag) + if key not in cache: # FIXME: what if the selector is "*" ? - elems = list(parent.findall(elem.tag)) - if elems[index] is elem: - yield elem - except (IndexError, KeyError): - pass + elems = parent.findall(elem.tag) + try: + cache[key] = elems[index] + except IndexError: + cache[key] = None + if cache[key] is elem: + yield elem return select raise SyntaxError("invalid predicate") diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 3e6871157d09299..5ef68640a09ba4b 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -752,7 +752,7 @@ def docroutine(self, object, name, mod=None, anchor = (cl and cl.__name__ or '') + '-' + name note = '' - title = '%s' % ( + title = '%s' % ( self.escape(anchor), self.escape(name)) if callable(object): @@ -766,13 +766,13 @@ def docroutine(self, object, name, mod=None, else: docstring = pydoc.getdoc(object) - decl = title + argspec + (note and self.grey( - '%s' % note)) + decl = title + argspec + (note and + '%s' % note) doc = self.markup( docstring, self.preformat, funcs, classes, methods) - doc = doc and '
    %s
    ' % doc - return '
    %s
    %s
    \n' % (decl, doc) + doc = doc and '
    %s
    ' % doc + return '
    %s
    %s
    \n' % (decl, doc) def docserver(self, server_name, package_documentation, methods): """Produce HTML documentation for an XML-RPC server.""" @@ -783,12 +783,11 @@ def docserver(self, server_name, package_documentation, methods): fdict[value] = fdict[key] server_name = self.escape(server_name) - head = '%s' % server_name - result = self.heading(head) + result = self.heading(server_name) doc = self.markup(package_documentation, self.preformat, fdict) - doc = doc and '%s' % doc - result = result + '

    %s

    \n' % doc + doc = doc and '
    %s
    \n' % doc + result = result + doc contents = [] method_items = sorted(methods.items()) @@ -807,12 +806,15 @@ def page(self, title, contents): '' % css_path) return '''\ - + + Python: %s -%s%s''' % (title, css_link, contents) +%s +%s +''' % (title, css_link, contents) class XMLRPCDocGenerator: """Generates documentation for an XML-RPC server. diff --git a/Makefile.pre.in b/Makefile.pre.in index b42a8491733c636..465981f8fd4deda 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -511,6 +511,7 @@ PYTHON_OBJS= \ Python/thread.o \ Python/traceback.o \ Python/tracemalloc.o \ + Python/typecache.o \ Python/uniqueid.o \ Python/getopt.o \ Python/pystrcmp.o \ @@ -1419,6 +1420,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_tracemalloc.h \ $(srcdir)/Include/internal/pycore_tstate.h \ $(srcdir)/Include/internal/pycore_tuple.h \ + $(srcdir)/Include/internal/pycore_typecache.h \ $(srcdir)/Include/internal/pycore_typedefs.h \ $(srcdir)/Include/internal/pycore_typeobject.h \ $(srcdir)/Include/internal/pycore_typevarobject.h \ diff --git a/Misc/NEWS.d/next/Build/2026-07-10-13-25-55.gh-issue-153511.dCPBwU.rst b/Misc/NEWS.d/next/Build/2026-07-10-13-25-55.gh-issue-153511.dCPBwU.rst new file mode 100644 index 000000000000000..0d9163c8dc4d0ee --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-07-10-13-25-55.gh-issue-153511.dCPBwU.rst @@ -0,0 +1,3 @@ +Removes the legacy MSI and Nuget build scripts, the legacy ``py.exe`` +launcher (now replaced by the Python install manager), and adds a +``PC/layout`` readme. diff --git a/Misc/NEWS.d/next/Build/2026-07-18-14-30-12.gh-issue-136687.teaPCG.rst b/Misc/NEWS.d/next/Build/2026-07-18-14-30-12.gh-issue-136687.teaPCG.rst new file mode 100644 index 000000000000000..a2c7a86f4430be7 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-07-18-14-30-12.gh-issue-136687.teaPCG.rst @@ -0,0 +1,4 @@ +Add the :option:`--with-curses` option to :program:`configure` to select the +curses backend (``ncursesw``, ``ncurses`` or the system's native ``curses``) or, +as :option:`--without-curses`, to exclude the :mod:`curses` and +:mod:`curses.panel` modules from the build. diff --git a/Misc/NEWS.d/next/Build/2026-07-19-07-48-43.gh-issue-154055.FemUJU.rst b/Misc/NEWS.d/next/Build/2026-07-19-07-48-43.gh-issue-154055.FemUJU.rst new file mode 100644 index 000000000000000..94fc9672ef7855f --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-07-19-07-48-43.gh-issue-154055.FemUJU.rst @@ -0,0 +1,4 @@ +The :mod:`curses` and :mod:`curses.panel` modules can now be built against a +curses library that lacks the X/Open ``attr_t`` and soft-label attribute +functions, ``scr_set()`` or ``resizeterm()`` -- such as the native SVr4 curses +of illumos and Solaris. These functions are probed and used only when present. diff --git a/Misc/NEWS.d/next/Build/2026-07-19-12-06-45.gh-issue-152902.0BXdab.rst b/Misc/NEWS.d/next/Build/2026-07-19-12-06-45.gh-issue-152902.0BXdab.rst new file mode 100644 index 000000000000000..c084d32664a186c --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-07-19-12-06-45.gh-issue-152902.0BXdab.rst @@ -0,0 +1,2 @@ +Added Intel ``icx`` compiler support to ``configure.ac``. Contributed by +High Performance Kernels LLC. diff --git a/Misc/NEWS.d/next/Build/2026-07-19-12-16-31.gh-issue-154070.f5da07.rst b/Misc/NEWS.d/next/Build/2026-07-19-12-16-31.gh-issue-154070.f5da07.rst new file mode 100644 index 000000000000000..31c6fd3ece35322 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-07-19-12-16-31.gh-issue-154070.f5da07.rst @@ -0,0 +1,4 @@ +Build the :mod:`curses` module against a wide-character capable ncurses +even when it is not named ``ncursesw`` -- for example the pkgsrc ncurses on +NetBSD and illumos, or the system ncurses on macOS. Such a library +previously produced a narrow build. diff --git a/Misc/NEWS.d/next/Build/2026-07-19-16-45-00.gh-issue-154136.mathpi.rst b/Misc/NEWS.d/next/Build/2026-07-19-16-45-00.gh-issue-154136.mathpi.rst new file mode 100644 index 000000000000000..9bfa313873893a4 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-07-19-16-45-00.gh-issue-154136.mathpi.rst @@ -0,0 +1,3 @@ +Fix building the :mod:`math` module on FreeBSD. +Define ``_ISOC23_SOURCE`` to make the C23 library declarations +(such as ``sinpi()`` in ``math.h``) visible. diff --git a/Misc/NEWS.d/next/C_API/2024-08-12-09-48-04.gh-issue-122931.x435Mb.rst b/Misc/NEWS.d/next/C_API/2024-08-12-09-48-04.gh-issue-122931.x435Mb.rst new file mode 100644 index 000000000000000..ff972aaefc74268 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-08-12-09-48-04.gh-issue-122931.x435Mb.rst @@ -0,0 +1 @@ +Allow importing stable ABI C extensions that include a multiarch tuple in their filename, e.g. ``foo.abi3-x86-64-linux-gnu.so``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-08-15-06-28-45.gh-issue-91484.huCgHt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-15-06-28-45.gh-issue-91484.huCgHt.rst new file mode 100644 index 000000000000000..7bba0da22d1c4da --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-08-15-06-28-45.gh-issue-91484.huCgHt.rst @@ -0,0 +1 @@ +:meth:`memoryview.cast` now allows casting from N-D to 1-D for F-contiguous. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-05-09-25-08.gh-issue-145685.afO33b.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-05-09-25-08.gh-issue-145685.afO33b.rst new file mode 100644 index 000000000000000..c95fc34b546aabb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-05-09-25-08.gh-issue-145685.afO33b.rst @@ -0,0 +1 @@ +The internal method cache used for types is now a per-type cache instead of a global per-interpreter one, improving performance and reducing cache misses. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-06-16-30-00.gh-issue-153236.lazy-import-errors.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-06-16-30-00.gh-issue-153236.lazy-import-errors.rst new file mode 100644 index 000000000000000..c5043af0a4f80b7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-06-16-30-00.gh-issue-153236.lazy-import-errors.rst @@ -0,0 +1,2 @@ +Propagate exceptions raised while importing lazy submodules instead of +reporting them as missing attributes. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-01-17-57.gh-issue-153419.U8HJOJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-01-17-57.gh-issue-153419.U8HJOJ.rst new file mode 100644 index 000000000000000..4459ec835395ea4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-01-17-57.gh-issue-153419.U8HJOJ.rst @@ -0,0 +1 @@ +Fix multiple :class:`bytearray` crashes and reference leaks caused by skipping :meth:`~object.__init__` and broken state setup code. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-14-57-19.gh-issue-153568.memoflt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-14-57-19.gh-issue-153568.memoflt.rst new file mode 100644 index 000000000000000..9e78e5ad545a83a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-14-57-19.gh-issue-153568.memoflt.rst @@ -0,0 +1,2 @@ +Speed up the parser by letting memoization lookups that cannot match return +immediately. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-07-49.gh-issue-153568.identcache.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-07-49.gh-issue-153568.identcache.rst new file mode 100644 index 000000000000000..5d778139d06b50d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-07-49.gh-issue-153568.identcache.rst @@ -0,0 +1 @@ +Speed up the parser by caching repeated identifiers during a parse. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-30-00.gh-issue-153570.Kb7Xd2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-30-00.gh-issue-153570.Kb7Xd2.rst new file mode 100644 index 000000000000000..655fd614d7f5514 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-11-15-30-00.gh-issue-153570.Kb7Xd2.rst @@ -0,0 +1,2 @@ +Fix a use-after-free in :meth:`bytearray.take_bytes` when the argument's +:meth:`~object.__index__` method resizes the bytearray. Patch by tonghuaroot. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-16-02-50-01.gh-issue-153785.fJqPKC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-16-02-50-01.gh-issue-153785.fJqPKC.rst new file mode 100644 index 000000000000000..40981acb37bbd70 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-16-02-50-01.gh-issue-153785.fJqPKC.rst @@ -0,0 +1,4 @@ +:exc:`AttributeError`: The default error message is now generated from ``name`` +and ``obj`` attributes when both are set and the exception was constructed with +no positional arguments, or with a single positional argument equal to ``name``. +Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-17-22-03-39.gh-issue-153881.oDa06s.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-17-22-03-39.gh-issue-153881.oDa06s.rst new file mode 100644 index 000000000000000..8facf88367fda7f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-17-22-03-39.gh-issue-153881.oDa06s.rst @@ -0,0 +1,2 @@ +Fix potential data race when calling :meth:`~object.__getstate__` +under the :term:`free-threaded build`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-33-27.gh-issue-153932.lKw2bo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-33-27.gh-issue-153932.lKw2bo.rst new file mode 100644 index 000000000000000..56b96307ed04cd9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-33-27.gh-issue-153932.lKw2bo.rst @@ -0,0 +1,2 @@ +Fix thread safety issue in the ``__reduce__`` method of +:py:class:`enumerate`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-16-18-25.gh-issue-154014.CJsjVL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-16-18-25.gh-issue-154014.CJsjVL.rst new file mode 100644 index 000000000000000..6bdcf83725168a2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-16-18-25.gh-issue-154014.CJsjVL.rst @@ -0,0 +1,2 @@ +Fix a JIT assertion during interpreter shutdown by initializing ``vm_data`` +fields for cold executors that bypass ``_Py_ExecutorInit()``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-19-14-39.gh-issue-154043.Kych7F.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-19-14-39.gh-issue-154043.Kych7F.rst new file mode 100644 index 000000000000000..d37e63e6116a593 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-19-14-39.gh-issue-154043.Kych7F.rst @@ -0,0 +1,2 @@ +Fix a data race when iterating a shared :class:`types.GenericAlias` iterator +from multiple threads under the :term:`free-threaded build`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-20-20-16-08.gh-issue-154275.l961cA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-20-20-16-08.gh-issue-154275.l961cA.rst new file mode 100644 index 000000000000000..998e91c77098e75 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-20-20-16-08.gh-issue-154275.l961cA.rst @@ -0,0 +1,2 @@ +Fix a crash when getting deeply nested ``__parameters__`` from a +:class:`types.GenericAlias` objects. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-22-14-30-00.gh-issue-154429.Qk9mZ2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-22-14-30-00.gh-issue-154429.Qk9mZ2.rst new file mode 100644 index 000000000000000..04781b03356b9fa --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-22-14-30-00.gh-issue-154429.Qk9mZ2.rst @@ -0,0 +1,2 @@ +Slightly speed up deallocation of objects that are not tracked by the garbage +collector, such as :class:`int`, :class:`float` and :class:`str`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-24-13-15-31.gh-issue-153809.JFA6GT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-24-13-15-31.gh-issue-153809.JFA6GT.rst new file mode 100644 index 000000000000000..d2d2c771a684325 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-24-13-15-31.gh-issue-153809.JFA6GT.rst @@ -0,0 +1,2 @@ +Fix interpreter crash while deallocating objects of :class:`asyncio.Task` on +free-threaded builds. Contributed by Sergey Miryanov. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-17-35-14.gh-issue-154695.aQWLLv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-17-35-14.gh-issue-154695.aQWLLv.rst new file mode 100644 index 000000000000000..6e147cf7f31a882 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-17-35-14.gh-issue-154695.aQWLLv.rst @@ -0,0 +1,2 @@ +Fix :class:`asyncio.Task` raising :exc:`AttributeError` when created with +``eager_start=True`` and no explicit *loop* argument. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-16-29-10.gh-issue-154775._ISRIk.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-16-29-10.gh-issue-154775._ISRIk.rst new file mode 100644 index 000000000000000..b0035c7e1bc0640 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-27-16-29-10.gh-issue-154775._ISRIk.rst @@ -0,0 +1,2 @@ +When matching a complex literal in :keyword:`case` statements, an extraneous +``+`` sign (for example, ``1++1j`` or ``1-+1j``) is no longer allowed. diff --git a/Misc/NEWS.d/next/Documentation/2026-07-17-22-33-43.gh-issue-118150.m7iFdP.rst b/Misc/NEWS.d/next/Documentation/2026-07-17-22-33-43.gh-issue-118150.m7iFdP.rst new file mode 100644 index 000000000000000..afa6007a46e1745 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2026-07-17-22-33-43.gh-issue-118150.m7iFdP.rst @@ -0,0 +1,2 @@ +Clarify in the :mod:`difflib` documentation what *junk* actually does, its +drawbacks, and how to control it. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-14-50-03.gh-issue-103925.khC-El.rst b/Misc/NEWS.d/next/Library/2023-04-27-14-50-03.gh-issue-103925.khC-El.rst new file mode 100644 index 000000000000000..858ce6e5bd98e70 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-27-14-50-03.gh-issue-103925.khC-El.rst @@ -0,0 +1,3 @@ +Fix :meth:`csv.Sniffer.sniff` for a sample with ``\r\n`` line endings in +which a quoted field ends a line: a letter could be detected as the +delimiter. diff --git a/Misc/NEWS.d/next/Library/2025-10-19-17-34-07.gh-issue-140326.kZM0pV.rst b/Misc/NEWS.d/next/Library/2025-10-19-17-34-07.gh-issue-140326.kZM0pV.rst new file mode 100644 index 000000000000000..07eea563ea5fbc0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-19-17-34-07.gh-issue-140326.kZM0pV.rst @@ -0,0 +1,3 @@ +Fix the :mod:`asyncio` REPL namespace so that relative imports no longer +resolve against the :mod:`asyncio` package and ``__file__`` is no longer +set. diff --git a/Misc/NEWS.d/next/Library/2026-02-22-11-27-57.gh-issue-145107.ZlQXEZ.rst b/Misc/NEWS.d/next/Library/2026-02-22-11-27-57.gh-issue-145107.ZlQXEZ.rst new file mode 100644 index 000000000000000..c15afb53d6db4a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-22-11-27-57.gh-issue-145107.ZlQXEZ.rst @@ -0,0 +1 @@ +Simplify ``asyncio.staggered_race`` by using ``eager_start=False``. diff --git a/Misc/NEWS.d/next/Library/2026-04-17-05-58-27.gh-issue-148665.AlUajE.rst b/Misc/NEWS.d/next/Library/2026-04-17-05-58-27.gh-issue-148665.AlUajE.rst new file mode 100644 index 000000000000000..db7736f1ae6ef65 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-17-05-58-27.gh-issue-148665.AlUajE.rst @@ -0,0 +1 @@ +socket.shutdown function is now enabled for Emscripten builds. However, if the runtime does not implement the shutdown syscall, it will show "Function not implemented" error. diff --git a/Misc/NEWS.d/next/Library/2026-04-18-12-00-00.gh-issue-148736.ag-frame.rst b/Misc/NEWS.d/next/Library/2026-04-18-12-00-00.gh-issue-148736.ag-frame.rst new file mode 100644 index 000000000000000..e9701dfaa6a48d5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-04-18-12-00-00.gh-issue-148736.ag-frame.rst @@ -0,0 +1,3 @@ +Fix a latent :exc:`AttributeError` in :mod:`asyncio` call-graph capture when +walking an async generator's ``ag_await`` chain: the walker referenced +``cr_frame`` instead of ``ag_frame``. diff --git a/Misc/NEWS.d/next/Library/2026-05-06-23-13-40.gh-issue-149319.PLM93t.rst b/Misc/NEWS.d/next/Library/2026-05-06-23-13-40.gh-issue-149319.PLM93t.rst new file mode 100644 index 000000000000000..4018ec43f5a47d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-06-23-13-40.gh-issue-149319.PLM93t.rst @@ -0,0 +1 @@ +The :mod:`asyncio` REPL now ignores :envvar:`PYTHONSTARTUP` and :envvar:`PYTHON_BASIC_REPL` when :option:`-E` or :option:`-I` is used. Patch by Jonathan Dung. diff --git a/Misc/NEWS.d/next/Library/2026-06-03-18-41-42.gh-issue-150866.gXGyX-.rst b/Misc/NEWS.d/next/Library/2026-06-03-18-41-42.gh-issue-150866.gXGyX-.rst new file mode 100644 index 000000000000000..57af6e954384b10 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-03-18-41-42.gh-issue-150866.gXGyX-.rst @@ -0,0 +1,2 @@ +Fix :meth:`asyncio.loop.shutdown_asyncgens` to report any +:exc:`BaseException` raised during asynchronous generator cleanup. diff --git a/Misc/NEWS.d/next/Library/2026-06-06-15-04-37.gh-issue-151022.1Aw8Tk.rst b/Misc/NEWS.d/next/Library/2026-06-06-15-04-37.gh-issue-151022.1Aw8Tk.rst new file mode 100644 index 000000000000000..f16f024c582fb3c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-06-15-04-37.gh-issue-151022.1Aw8Tk.rst @@ -0,0 +1,2 @@ +Fix ``_remote_debugging`` stack traces for code objects with large line +tables. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-10-00-00.gh-issue-119710.Qz7Kp2.rst b/Misc/NEWS.d/next/Library/2026-06-11-10-00-00.gh-issue-119710.Qz7Kp2.rst new file mode 100644 index 000000000000000..45f99d72ea29a2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-10-00-00.gh-issue-119710.Qz7Kp2.rst @@ -0,0 +1,4 @@ +Fix :mod:`asyncio` subprocess :meth:`~asyncio.subprocess.Process.wait` +hanging when the process has exited but one of its pipes is kept open by an +inherited child process (so the pipe never reaches EOF). ``wait()`` now +returns as soon as the process exits, regardless of the pipes' state. diff --git a/Misc/NEWS.d/next/Library/2026-06-21-00-00-57.gh-issue-151822.bOC2G56F.rst b/Misc/NEWS.d/next/Library/2026-06-21-00-00-57.gh-issue-151822.bOC2G56F.rst new file mode 100644 index 000000000000000..70e7323c7e2ef6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-21-00-00-57.gh-issue-151822.bOC2G56F.rst @@ -0,0 +1 @@ +Colorize ``exit``, ``quit``, ``copyright``, ``help``, and ``clear`` as commands in the :term:`REPL` when typed alone on a line. Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Library/2026-07-03-02-11-00.gh-issue-152912.Mv3KpR.rst b/Misc/NEWS.d/next/Library/2026-07-03-02-11-00.gh-issue-152912.Mv3KpR.rst new file mode 100644 index 000000000000000..a4d7c4a9b0a2948 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-03-02-11-00.gh-issue-152912.Mv3KpR.rst @@ -0,0 +1 @@ +``sys.addaudithook()`` now correctly suppresses only :exc:`RuntimeError` instead of all :exc:`Exception` subclasses when an existing audit hook raises during hook registration. Patch by Yeongu Kim. diff --git a/Misc/NEWS.d/next/Library/2026-07-08-00-00-00.gh-issue-153291.rdlnCS.rst b/Misc/NEWS.d/next/Library/2026-07-08-00-00-00.gh-issue-153291.rdlnCS.rst new file mode 100644 index 000000000000000..dbe3cd417e0d607 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-08-00-00-00.gh-issue-153291.rdlnCS.rst @@ -0,0 +1,4 @@ +Fix a data race in :func:`readline.get_completer` and +:func:`readline.get_pre_input_hook` on the :term:`free-threaded ` build: the getters read the stored hook without the critical +section that the corresponding setters hold. diff --git a/Misc/NEWS.d/next/Library/2026-07-08-15-26-00.gh-issue-153296.Twix12.rst b/Misc/NEWS.d/next/Library/2026-07-08-15-26-00.gh-issue-153296.Twix12.rst new file mode 100644 index 000000000000000..22604dd31dd6050 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-08-15-26-00.gh-issue-153296.Twix12.rst @@ -0,0 +1 @@ +Fix a data race and use-after-free when iterating over an :class:`io.StringIO` object while it is being concurrently mutated. The ``__next__`` method now properly acquires the object's lock. diff --git a/Misc/NEWS.d/next/Library/2026-07-09-08-40-00.gh-issue-153290.GTzSEa.rst b/Misc/NEWS.d/next/Library/2026-07-09-08-40-00.gh-issue-153290.GTzSEa.rst new file mode 100644 index 000000000000000..deea3fe66b9b5af --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-08-40-00.gh-issue-153290.GTzSEa.rst @@ -0,0 +1,3 @@ +Fix a data race on the free-threaded build when :meth:`!io.BytesIO.__setstate__` +installs the instance dictionary while another thread concurrently calls a +method on the same object. diff --git a/Misc/NEWS.d/next/Library/2026-07-14-12-00-00.gh-issue-83273.SnifQ7.rst b/Misc/NEWS.d/next/Library/2026-07-14-12-00-00.gh-issue-83273.SnifQ7.rst new file mode 100644 index 000000000000000..789503913048f94 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-14-12-00-00.gh-issue-83273.SnifQ7.rst @@ -0,0 +1,10 @@ +:meth:`csv.Sniffer.sniff` now deduces the dialect by trial parsing with the +actual CSV parser instead of heuristics based on matching isolated fragments +and on character frequencies. It can now +detect the *escapechar* parameter, accepts non-ASCII delimiters in the +*delimiters* argument, no longer misdetects the delimiter when the sample +contains delimiter characters inside quoted fields, handles samples +truncated at an arbitrary point, and no longer takes quadratic time on +quoted samples. A sample consisting of a single column of quoted fields +now raises :exc:`csv.Error` instead of guessing a delimiter from the +content of the fields. diff --git a/Misc/NEWS.d/next/Library/2026-07-14-15-00-00.gh-issue-127049.pQw7Rk.rst b/Misc/NEWS.d/next/Library/2026-07-14-15-00-00.gh-issue-127049.pQw7Rk.rst new file mode 100644 index 000000000000000..d7557d8f562f7a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-14-15-00-00.gh-issue-127049.pQw7Rk.rst @@ -0,0 +1,5 @@ +Fix a race condition in :mod:`asyncio` on Unix where +:meth:`asyncio.subprocess.Process.send_signal`, :meth:`~asyncio.subprocess.Process.terminate` +or :meth:`~asyncio.subprocess.Process.kill` could signal an unrelated process +that was recycled onto the PID of the already-reaped child when ThreadedChildWatcher is used. +Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2026-07-15-21-56-40.gh-issue-146011.nWmHif.rst b/Misc/NEWS.d/next/Library/2026-07-15-21-56-40.gh-issue-146011.nWmHif.rst new file mode 100644 index 000000000000000..0cac0257040bd6b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-15-21-56-40.gh-issue-146011.nWmHif.rst @@ -0,0 +1,2 @@ +Fix a heap-use-after-free in the C implementation of :mod:`decimal` +when calling :func:`repr` after deleting the :class:`~decimal.Context`. diff --git a/Misc/NEWS.d/next/Library/2026-07-17-14-30-00.gh-issue-153856.JEHepm.rst b/Misc/NEWS.d/next/Library/2026-07-17-14-30-00.gh-issue-153856.JEHepm.rst new file mode 100644 index 000000000000000..f0af3de70848339 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-17-14-30-00.gh-issue-153856.JEHepm.rst @@ -0,0 +1 @@ +Add :data:`os.RWF_NOSIGNAL` constant for Linux 6.18+. diff --git a/Misc/NEWS.d/next/Library/2026-07-17-20-31-00.gh-issue-153862.X3kjoY.rst b/Misc/NEWS.d/next/Library/2026-07-17-20-31-00.gh-issue-153862.X3kjoY.rst new file mode 100644 index 000000000000000..aa646dc6c49174b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-17-20-31-00.gh-issue-153862.X3kjoY.rst @@ -0,0 +1,3 @@ +On a wide :mod:`curses` build, :meth:`curses.window.inch` now returns the +locale-encoded byte of a non-ASCII character, matching +:meth:`~curses.window.instr`, instead of the low byte of its code point. diff --git a/Misc/NEWS.d/next/Library/2026-07-17-20-56-32.gh-issue-153864.WmA9By.rst b/Misc/NEWS.d/next/Library/2026-07-17-20-56-32.gh-issue-153864.WmA9By.rst new file mode 100644 index 000000000000000..79857185d7d0c76 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-17-20-56-32.gh-issue-153864.WmA9By.rst @@ -0,0 +1,3 @@ +On a wide :mod:`curses` build, :meth:`curses.window.insch` now inserts a +non-ASCII byte as the character it encodes in the window's encoding, +consistently with :meth:`~curses.window.addch`, instead of its code point. diff --git a/Misc/NEWS.d/next/Library/2026-07-18-06-16-55.gh-issue-153903.mJFrs8.rst b/Misc/NEWS.d/next/Library/2026-07-18-06-16-55.gh-issue-153903.mJFrs8.rst new file mode 100644 index 000000000000000..00f89b471175f45 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-06-16-55.gh-issue-153903.mJFrs8.rst @@ -0,0 +1,2 @@ +Add :func:`ctypes.util.wrap_dll_function` for creating +:class:`~ctypes._CFuncPtr` objects from a function signature. diff --git a/Misc/NEWS.d/next/Library/2026-07-18-10-52-10.gh-issue-153896.87oevp.rst b/Misc/NEWS.d/next/Library/2026-07-18-10-52-10.gh-issue-153896.87oevp.rst new file mode 100644 index 000000000000000..217a3d3d272ed07 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-10-52-10.gh-issue-153896.87oevp.rst @@ -0,0 +1 @@ +Deduplicate unhashable args in :data:`typing.Literal`. diff --git a/Misc/NEWS.d/next/Library/2026-07-18-11-16-41.gh-issue-153908.82FiGk.rst b/Misc/NEWS.d/next/Library/2026-07-18-11-16-41.gh-issue-153908.82FiGk.rst new file mode 100644 index 000000000000000..6ee943a5fb66724 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-11-16-41.gh-issue-153908.82FiGk.rst @@ -0,0 +1 @@ +Fix data race when calling :func:`repr` on :class:`itertools.count` under the :term:`free-threaded build`. diff --git a/Misc/NEWS.d/next/Library/2026-07-18-13-30-00.gh-issue-153906.mHtNdY.rst b/Misc/NEWS.d/next/Library/2026-07-18-13-30-00.gh-issue-153906.mHtNdY.rst new file mode 100644 index 000000000000000..487587145f1a3eb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-13-30-00.gh-issue-153906.mHtNdY.rst @@ -0,0 +1,4 @@ +Modernize the HTML output of :mod:`pydoc`: use semantic HTML5 markup and +a new style sheet based on the python-docs-theme used by docs.python.org, +with dark mode support. Class members inherited from other classes are +now collapsed by default. diff --git a/Misc/NEWS.d/next/Library/2026-07-18-17-09-49.gh-issue-154001.3brrUv.rst b/Misc/NEWS.d/next/Library/2026-07-18-17-09-49.gh-issue-154001.3brrUv.rst new file mode 100644 index 000000000000000..ff019aa36188479 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-17-09-49.gh-issue-154001.3brrUv.rst @@ -0,0 +1,2 @@ +Fix :func:`random.binomialvariate` raising :exc:`ZeroDivisionError` +when :func:`random.random` returns zero. diff --git a/Misc/NEWS.d/next/Library/2026-07-19-00-00-00.gh-issue-154053.LibreSSLsigalgs.rst b/Misc/NEWS.d/next/Library/2026-07-19-00-00-00.gh-issue-154053.LibreSSLsigalgs.rst new file mode 100644 index 000000000000000..3d453eb7314345c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-00-00-00.gh-issue-154053.LibreSSLsigalgs.rst @@ -0,0 +1,4 @@ +Fix compilation of the :mod:`ssl` module against LibreSSL, which does not +provide ``SSL_CTX_set1_sigalgs_list()`` and ``SSL_CTX_set1_client_sigalgs_list()``. +The :meth:`!set_server_sigalgs` and :meth:`!set_client_sigalgs` methods of +:class:`ssl.SSLContext` now raise :exc:`NotImplementedError` on LibreSSL. diff --git a/Misc/NEWS.d/next/Library/2026-07-19-19-05-00.gh-issue-154146.acospi.rst b/Misc/NEWS.d/next/Library/2026-07-19-19-05-00.gh-issue-154146.acospi.rst new file mode 100644 index 000000000000000..5dbddbadee0bd3d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-19-05-00.gh-issue-154146.acospi.rst @@ -0,0 +1,2 @@ +Fix accuracy of :func:`math.acospi` for arguments close to 1 +on platforms that do not provide ``acospi()`` in libm. diff --git a/Misc/NEWS.d/next/Library/2026-07-19-21-30-00.gh-issue-154176.strxfrm.rst b/Misc/NEWS.d/next/Library/2026-07-19-21-30-00.gh-issue-154176.strxfrm.rst new file mode 100644 index 000000000000000..9ffa5828000113b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-21-30-00.gh-issue-154176.strxfrm.rst @@ -0,0 +1,2 @@ +Fix a crash in :func:`locale.strxfrm` on DragonFly BSD, +whose ``wcsxfrm()`` does not support the zero size. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-12-40-00.gh-issue-145030.TFJm6k.rst b/Misc/NEWS.d/next/Library/2026-07-20-12-40-00.gh-issue-145030.TFJm6k.rst new file mode 100644 index 000000000000000..2c6b07a4fa8b76f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-12-40-00.gh-issue-145030.TFJm6k.rst @@ -0,0 +1,3 @@ +Fix :mod:`asyncio` write pipe transports for named FIFOs on macOS. Unread +data sitting in the FIFO made the transport misinterpret a poll event as +the reader disconnecting, wrongly closing the transport. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-12-45-00.gh-issue-139373.qL7xKm.rst b/Misc/NEWS.d/next/Library/2026-07-20-12-45-00.gh-issue-139373.qL7xKm.rst new file mode 100644 index 000000000000000..be7d5b79f099e03 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-12-45-00.gh-issue-139373.qL7xKm.rst @@ -0,0 +1,3 @@ +Fix :meth:`asyncio.subprocess.Process.communicate` losing already-read +output when it is cancelled; the output is now retained and returned by a +subsequent ``communicate()`` call. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-14-00-00.gh-issue-154227.openpt.rst b/Misc/NEWS.d/next/Library/2026-07-20-14-00-00.gh-issue-154227.openpt.rst new file mode 100644 index 000000000000000..e33d090d78c5934 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-14-00-00.gh-issue-154227.openpt.rst @@ -0,0 +1,2 @@ +Fix :func:`os.posix_openpt` on OpenBSD, where it rejected the +:data:`~os.O_CLOEXEC` flag. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-14-10-00.gh-issue-154225.openpty.rst b/Misc/NEWS.d/next/Library/2026-07-20-14-10-00.gh-issue-154225.openpty.rst new file mode 100644 index 000000000000000..f15022b66320062 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-14-10-00.gh-issue-154225.openpty.rst @@ -0,0 +1,2 @@ +Fix :func:`os.openpty` on Solaris and illumos: it no longer leaves the +pseudo-terminal as the controlling terminal of the calling process. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-18-30-00.gh-issue-154258.mmapresize.rst b/Misc/NEWS.d/next/Library/2026-07-20-18-30-00.gh-issue-154258.mmapresize.rst new file mode 100644 index 000000000000000..4c33336e0bbdcad --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-18-30-00.gh-issue-154258.mmapresize.rst @@ -0,0 +1,3 @@ +Fix a crash in :meth:`mmap.mmap.resize` on NetBSD when growing a shared +anonymous mapping. :meth:`!resize` now raises :exc:`ValueError` in this +case, as it already did on Linux. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-21-36-40.gh-issue-154283.Xip7xE.rst b/Misc/NEWS.d/next/Library/2026-07-20-21-36-40.gh-issue-154283.Xip7xE.rst new file mode 100644 index 000000000000000..a278d67e1824d58 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-21-36-40.gh-issue-154283.Xip7xE.rst @@ -0,0 +1,2 @@ +On DragonFly BSD, :func:`threading.get_native_id` now returns a value that is +unique across processes, matching the other platforms. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-22-03-44.gh-issue-154291.Mi5HeQ.rst b/Misc/NEWS.d/next/Library/2026-07-20-22-03-44.gh-issue-154291.Mi5HeQ.rst new file mode 100644 index 000000000000000..d962f8987ad491e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-22-03-44.gh-issue-154291.Mi5HeQ.rst @@ -0,0 +1,3 @@ +Fix :func:`socket.has_dualstack_ipv6` to return ``False`` on platforms such as +DragonFly BSD where setting ``IPV6_V6ONLY`` to 0 silently has no +effect. diff --git a/Misc/NEWS.d/next/Library/2026-07-21-01-13-39.gh-issue-154307.UNFKL4.rst b/Misc/NEWS.d/next/Library/2026-07-21-01-13-39.gh-issue-154307.UNFKL4.rst new file mode 100644 index 000000000000000..fdba0c2940f0050 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-21-01-13-39.gh-issue-154307.UNFKL4.rst @@ -0,0 +1,3 @@ +Fix :meth:`tempfile.TemporaryDirectory.cleanup` on DragonFly BSD, where removing +a file with the ``UF_NOUNLINK`` flag failed with ``EISDIR`` instead of +``EPERM``. diff --git a/Misc/NEWS.d/next/Library/2026-07-21-20-34-05.gh-issue-154389.DlTiii.rst b/Misc/NEWS.d/next/Library/2026-07-21-20-34-05.gh-issue-154389.DlTiii.rst new file mode 100644 index 000000000000000..eb2e2edc2d720e7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-21-20-34-05.gh-issue-154389.DlTiii.rst @@ -0,0 +1,3 @@ +Fix :func:`uuid.uuid1` on OpenBSD: +it returned a version 4 UUID, +because ``uuid_create()`` generates random UUIDs on this platform. diff --git a/Misc/NEWS.d/next/Library/2026-07-21-21-36-59.gh-issue-154399.VSa6oH.rst b/Misc/NEWS.d/next/Library/2026-07-21-21-36-59.gh-issue-154399.VSa6oH.rst new file mode 100644 index 000000000000000..598dd26694c73e4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-21-21-36-59.gh-issue-154399.VSa6oH.rst @@ -0,0 +1,3 @@ +Fix :mod:`venv` activation in a non-interactive csh: +``activate.csh`` no longer fails +when the ``prompt`` variable is not set. diff --git a/Misc/NEWS.d/next/Library/2026-07-22-10-30-08.gh-issue-154435.xCxm0T.rst b/Misc/NEWS.d/next/Library/2026-07-22-10-30-08.gh-issue-154435.xCxm0T.rst new file mode 100644 index 000000000000000..b468e929b1ca18b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-22-10-30-08.gh-issue-154435.xCxm0T.rst @@ -0,0 +1,3 @@ +Fix :func:`os.posix_fadvise` and :func:`os.posix_fallocate` on DragonFly BSD: +they raised :exc:`OSError` with a meaningless error code, +because these functions return -1 and set ``errno`` there. diff --git a/Misc/NEWS.d/next/Library/2026-07-22-12-59-26.gh-issue-154467.AA9w1D.rst b/Misc/NEWS.d/next/Library/2026-07-22-12-59-26.gh-issue-154467.AA9w1D.rst new file mode 100644 index 000000000000000..9d7ac18a5e9db12 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-22-12-59-26.gh-issue-154467.AA9w1D.rst @@ -0,0 +1,3 @@ +Fixed :mod:`pdb` remote attaching (``python -m pdb -p PID``) sending an +empty prompt to the client instead of ``(Pdb)`` when the target process has +an interactive terminal. diff --git a/Misc/NEWS.d/next/Library/2026-07-22-13-00-00.gh-issue-154460.wCsFtM.rst b/Misc/NEWS.d/next/Library/2026-07-22-13-00-00.gh-issue-154460.wCsFtM.rst new file mode 100644 index 000000000000000..b0a7058f0519ff0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-22-13-00-00.gh-issue-154460.wCsFtM.rst @@ -0,0 +1,2 @@ +Fix :func:`time.strftime` and :meth:`datetime.datetime.strftime` returning a +wrong ISO 8601 week number (``%V``) on OpenBSD. diff --git a/Misc/NEWS.d/next/Library/2026-07-22-13-01-24.gh-issue-73458.byxSPi.rst b/Misc/NEWS.d/next/Library/2026-07-22-13-01-24.gh-issue-73458.byxSPi.rst new file mode 100644 index 000000000000000..27b6eeac5b520b2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-22-13-01-24.gh-issue-73458.byxSPi.rst @@ -0,0 +1,5 @@ +Fix :func:`logging.config.listen`: it left the caller waiting for the +``ready`` event forever if the server could not be started, +for example if the port was invalid or already in use. +It now also binds to an IPv6 address if the host has no IPv4 address, +for example if ``localhost`` is only aliased to ``::1``. diff --git a/Misc/NEWS.d/next/Library/2026-07-22-15-56-11.gh-issue-154189.7zWWNZ.rst b/Misc/NEWS.d/next/Library/2026-07-22-15-56-11.gh-issue-154189.7zWWNZ.rst new file mode 100644 index 000000000000000..3a745a7c5ce67d1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-22-15-56-11.gh-issue-154189.7zWWNZ.rst @@ -0,0 +1,4 @@ +Fixed a potential use-after-free when calling :func:`functools.partial`. +Now, when invoking a :func:`~functools.partial` object, the stored function, +positional arguments, and keyword arguments are preserved for the duration +of the call in case of reentrancy. diff --git a/Misc/NEWS.d/next/Library/2026-07-22-17-35-24.gh-issue-70990.xVHKwt.rst b/Misc/NEWS.d/next/Library/2026-07-22-17-35-24.gh-issue-70990.xVHKwt.rst new file mode 100644 index 000000000000000..4af22f88ddc413f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-22-17-35-24.gh-issue-70990.xVHKwt.rst @@ -0,0 +1,4 @@ +:class:`logging.handlers.SysLogHandler` now accepts a :class:`bytes` address +of a Unix domain socket, including an address in the abstract namespace. +Previously only :class:`str` was recognized, +and a :class:`bytes` address raised :exc:`ValueError`. diff --git a/Misc/NEWS.d/next/Library/2026-07-23-13-09-25.gh-issue-154199.hmBfrS.rst b/Misc/NEWS.d/next/Library/2026-07-23-13-09-25.gh-issue-154199.hmBfrS.rst new file mode 100644 index 000000000000000..37463944a5de51b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-23-13-09-25.gh-issue-154199.hmBfrS.rst @@ -0,0 +1,2 @@ +Make :func:`ctypes.util.find_msvcrt` always return ``None`` for +compatibility reasons. diff --git a/Misc/NEWS.d/next/Library/2026-07-23-19-45-00.gh-issue-154551.Ct7pL9.rst b/Misc/NEWS.d/next/Library/2026-07-23-19-45-00.gh-issue-154551.Ct7pL9.rst new file mode 100644 index 000000000000000..c4db19d89364e07 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-23-19-45-00.gh-issue-154551.Ct7pL9.rst @@ -0,0 +1 @@ +Fix :func:`ctypes.util.find_library` returning ``None`` in non-UTF-8 locales. diff --git a/Misc/NEWS.d/next/Library/2026-07-24-15-54-35.gh-issue-142035.ZpNfl6.rst b/Misc/NEWS.d/next/Library/2026-07-24-15-54-35.gh-issue-142035.ZpNfl6.rst new file mode 100644 index 000000000000000..f11f3a9e55200d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-24-15-54-35.gh-issue-142035.ZpNfl6.rst @@ -0,0 +1 @@ +Fix incorrect wrapping of :mod:`argparse` help text when color is enabled. diff --git a/Misc/NEWS.d/next/Library/2026-07-24-17-02-46.gh-issue-154638.7YnFHH.rst b/Misc/NEWS.d/next/Library/2026-07-24-17-02-46.gh-issue-154638.7YnFHH.rst new file mode 100644 index 000000000000000..91c22640deba6b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-24-17-02-46.gh-issue-154638.7YnFHH.rst @@ -0,0 +1 @@ +Improve import time of :mod:`argparse` by lazily importing several dependencies. diff --git a/Misc/NEWS.d/next/Library/2026-07-25-13-30-00.gh-issue-93251.gAiErr.rst b/Misc/NEWS.d/next/Library/2026-07-25-13-30-00.gh-issue-93251.gAiErr.rst new file mode 100644 index 000000000000000..3f313030802f875 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-25-13-30-00.gh-issue-93251.gAiErr.rst @@ -0,0 +1,4 @@ +Fix :exc:`UnicodeDecodeError` in :mod:`socket` functions +(such as :func:`~socket.getaddrinfo` and :func:`~socket.gethostbyaddr`) +when the localized error message of the C library is not UTF-8: +decode it from the locale encoding. diff --git a/Misc/NEWS.d/next/Library/2026-07-26-13-40-00.gh-issue-154744.Sk1Psp.rst b/Misc/NEWS.d/next/Library/2026-07-26-13-40-00.gh-issue-154744.Sk1Psp.rst new file mode 100644 index 000000000000000..500826c195346f7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-26-13-40-00.gh-issue-154744.Sk1Psp.rst @@ -0,0 +1 @@ +Improve detection of ``skipinitialspace`` in :meth:`csv.Sniffer.sniff`. diff --git a/Misc/NEWS.d/next/Security/2026-06-30-13-24-13.gh-issue-152674.-2QVoL.rst b/Misc/NEWS.d/next/Security/2026-06-30-13-24-13.gh-issue-152674.-2QVoL.rst new file mode 100644 index 000000000000000..69e73008af79f14 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-30-13-24-13.gh-issue-152674.-2QVoL.rst @@ -0,0 +1,6 @@ +The :class:`xml.etree.ElementTree.Element` methods +:meth:`~xml.etree.ElementTree.Element.findall`, +:meth:`~xml.etree.ElementTree.Element.iterfind` and +:meth:`~xml.etree.ElementTree.Element.find` avoid quadratic behavior when +using XPath index predicates (``[1]``, ``[last()]``, ``[last()-N]``) on XML +documents with many same-tag siblings. diff --git a/Misc/NEWS.d/next/Tests/2026-06-29-10-14-09.gh-issue-152548.Khw9J7.rst b/Misc/NEWS.d/next/Tests/2026-06-29-10-14-09.gh-issue-152548.Khw9J7.rst new file mode 100644 index 000000000000000..e8c81ed8a1f7ca9 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-06-29-10-14-09.gh-issue-152548.Khw9J7.rst @@ -0,0 +1,3 @@ +Add the :func:`test.support.isolation.runInSubprocess` decorator +to run a test method or ``TestCase`` subclass in a fresh interpreter subprocess, +isolated from the rest of the test run. diff --git a/Misc/NEWS.d/next/Tests/2026-07-19-18-20-00.gh-issue-154144.tcbool.rst b/Misc/NEWS.d/next/Tests/2026-07-19-18-20-00.gh-issue-154144.tcbool.rst new file mode 100644 index 000000000000000..dc1ac8b4035153c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-07-19-18-20-00.gh-issue-154144.tcbool.rst @@ -0,0 +1 @@ +Fix building the :mod:`!_testcapi` module on NetBSD. diff --git a/Misc/NEWS.d/next/Tests/2026-07-19-20-15-00.gh-issue-154167.sigint.rst b/Misc/NEWS.d/next/Tests/2026-07-19-20-15-00.gh-issue-154167.sigint.rst new file mode 100644 index 000000000000000..b2620f4a0bee54c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-07-19-20-15-00.gh-issue-154167.sigint.rst @@ -0,0 +1,3 @@ +The test runner (regrtest) now restores the default SIGINT handler if it +was inherited as ignored, so the test suite no longer hangs when run as +a shell background job. diff --git a/Misc/NEWS.d/next/Tests/2026-07-20-11-30-00.gh-issue-154211.hugestack.rst b/Misc/NEWS.d/next/Tests/2026-07-20-11-30-00.gh-issue-154211.hugestack.rst new file mode 100644 index 000000000000000..8030db869c7ba6e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-07-20-11-30-00.gh-issue-154211.hugestack.rst @@ -0,0 +1,3 @@ +Add ``test.support.skip_if_huge_c_stack()`` and use it to skip tests +that exhaust the C stack if the stack limit is very large +(e.g. on DragonFly BSD or with ``ulimit -s unlimited``). diff --git a/Misc/NEWS.d/next/Tests/2026-07-24-12-00-00.gh-issue-76595.KfF5xR.rst b/Misc/NEWS.d/next/Tests/2026-07-24-12-00-00.gh-issue-76595.KfF5xR.rst new file mode 100644 index 000000000000000..c344ee2f2cb30b7 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-07-24-12-00-00.gh-issue-76595.KfF5xR.rst @@ -0,0 +1 @@ +Add C API tests for :c:func:`PyCapsule_Import`. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-07-24-09-30-00.gh-issue-154580.Kp3nQ2.rst b/Misc/NEWS.d/next/Tools-Demos/2026-07-24-09-30-00.gh-issue-154580.Kp3nQ2.rst new file mode 100644 index 000000000000000..f6a039245b015ed --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-07-24-09-30-00.gh-issue-154580.Kp3nQ2.rst @@ -0,0 +1,3 @@ +Fix ``python-gdb.py`` raising :exc:`UnicodeEncodeError` when pretty-printing a +non-ASCII :class:`str` in a locale whose host charset cannot encode it, such as +any non-ASCII string in the C locale. diff --git a/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst b/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst deleted file mode 100644 index 9a57536f1dc96b6..000000000000000 --- a/Misc/NEWS.d/next/Windows/2026-05-06-21-36-53.gh-issue-124111.m4OBX8.rst +++ /dev/null @@ -1 +0,0 @@ -Updated Windows builds to use Tcl/Tk 9.0.3. diff --git a/Misc/NEWS.d/next/Windows/2026-06-28-12-37-08.gh-issue-152433.FTMgqF.rst b/Misc/NEWS.d/next/Windows/2026-06-28-12-37-08.gh-issue-152433.FTMgqF.rst new file mode 100644 index 000000000000000..bd295fb4dd603f5 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-06-28-12-37-08.gh-issue-152433.FTMgqF.rst @@ -0,0 +1 @@ +Implement :func:`os.cpu_count` for Universal Windows Platform. diff --git a/Misc/NEWS.d/next/Windows/2026-06-30-16-40-00.gh-issue-152433.VaeEwR.rst b/Misc/NEWS.d/next/Windows/2026-06-30-16-40-00.gh-issue-152433.VaeEwR.rst new file mode 100644 index 000000000000000..00f89ffcbc1c29a --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-06-30-16-40-00.gh-issue-152433.VaeEwR.rst @@ -0,0 +1 @@ +``_winapi``: implement ``GetVersion()`` UWP alternative. diff --git a/Misc/NEWS.d/next/Windows/2026-07-17-16-21-17.gh-issue-152433.45rR2A.rst b/Misc/NEWS.d/next/Windows/2026-07-17-16-21-17.gh-issue-152433.45rR2A.rst new file mode 100644 index 000000000000000..eb45c1cbd0e9ea7 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-07-17-16-21-17.gh-issue-152433.45rR2A.rst @@ -0,0 +1,2 @@ +Modernize fileutils removing GetFileInformationByHandle API calls and allow +build for Universal Windows Platform. diff --git a/Misc/NEWS.d/next/Windows/2026-07-18-11-56-53.gh-issue-124111.m4OBX8.rst b/Misc/NEWS.d/next/Windows/2026-07-18-11-56-53.gh-issue-124111.m4OBX8.rst new file mode 100644 index 000000000000000..802afc6f7a6fc48 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-07-18-11-56-53.gh-issue-124111.m4OBX8.rst @@ -0,0 +1 @@ +Updated Windows builds to use Tcl/Tk 9.0.4. diff --git a/Misc/NEWS.d/next/Windows/2026-07-21-10-42-18.gh-issue-69573.Kv7QpX.rst b/Misc/NEWS.d/next/Windows/2026-07-21-10-42-18.gh-issue-69573.Kv7QpX.rst new file mode 100644 index 000000000000000..441718032cc0086 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-07-21-10-42-18.gh-issue-69573.Kv7QpX.rst @@ -0,0 +1,4 @@ +:func:`msvcrt.putch` and :func:`msvcrt.putwch` now check the return value of +the underlying ``_putch()`` and ``_putwch()`` C functions and raise +:exc:`OSError` on failure, for example when the process has no console +attached, instead of silently ignoring the error. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 523d20259adaaa9..cdacaf6f5869531 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -112,42 +112,42 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "7a1d1f3a2b8f4484a9c2a027a157963c18f85a81785e85fcb5d1e3df6b6a4fd4" + "checksumValue": "3ac2acd65ddaaac0b2b8df321b558419d21da20bf67ab39149b8248c85f0d214" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-9.0.3.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tcl-9.0.4.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.4.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tcl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "9.0.3.0" + "versionInfo": "9.0.4.0" }, { "SPDXID": "SPDXRef-PACKAGE-tk", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "54fb59df12c489c6264f5b7d3d7444b150d1e3d6561fd59cdb11483440cec000" + "checksumValue": "66048966cfa88989333ff1632f454e10e9248516bb45efda7b4d2ae5a61642cb" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-9.0.3.1.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/tk-9.0.4.1.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.3.1:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:tcl_tk:tcl_tk:9.0.4.1:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "tk", "primaryPackagePurpose": "SOURCE", - "versionInfo": "9.0.3.1" + "versionInfo": "9.0.4.1" }, { "SPDXID": "SPDXRef-PACKAGE-xz", diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 524f1466b191f64..eaf8777a56c59b6 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -172,9 +172,9 @@ @MODULE_XXSUBTYPE_TRUE@xxsubtype xxsubtype.c @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c -@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c +@MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c _testinternalcapi/interpreter.c _testinternalcapi/tuple.c _testinternalcapi/typecache.c @MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/modsupport.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/module.c _testcapi/weakref.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/capsule.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/slots.c _testlimitedcapi/sys.c _testlimitedcapi/threadstate.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index bc0f7f3901e0eda..0cd41d0b4c4d0dc 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -2364,7 +2364,9 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, _PyObject_SetMaybeWeakref((PyObject *)self); #endif if (eager_start) { - PyObject *res = PyObject_CallMethodNoArgs(loop, &_Py_ID(is_running)); + // gh-154695: loop may be None here, future_init() resolved it into task_loop. + PyObject *res = PyObject_CallMethodNoArgs(self->task_loop, + &_Py_ID(is_running)); if (res == NULL) { return -1; } @@ -2966,13 +2968,17 @@ TaskObj_dealloc(PyObject *self) if (PyObject_CallFinalizerFromDealloc(self) < 0) { return; // resurrected } + // Untrack the object before unregistering the task, since the + // latter can cause a stop-the-world pause, after which another + // thread might call the GC and reach the object while it is + // semi-deallocated but still tracked + PyObject_GC_UnTrack(self); + // unregister the task after finalization so that // if the task gets resurrected, it remains registered unregister_task((TaskObj *)self); PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_UnTrack(self); - PyObject_ClearWeakRefs(self); (void)TaskObj_clear(self); diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index cc68a51f0bed935..01ea3c43cce2e6f 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -356,6 +356,35 @@ _PyCursesStatefulCheckFunction(PyObject *module, return 0; } +/* + * Function to check that the current screen has a terminal, by testing + * stdscr, which a screen made by new_prescr() does not have. If an error + * occurs, a PyCursesError is set and this returns 0. Otherwise this + * returns 1. + */ +static int +_PyCursesStatefulCheckTerminal(PyObject *module) +{ + if (stdscr != NULL) { + return 1; + } + cursesmodule_state *state = get_cursesmodule_state(module); + PyErr_SetString(state->error, "the current screen has no terminal"); + return 0; +} + +/* Same as _PyCursesStatefulCheckTerminal() for a Window object. */ +static int +curses_window_check_terminal(PyCursesWindowObject *win) +{ + if (stdscr != NULL) { + return 1; + } + cursesmodule_state *state = get_cursesmodule_state_by_win(win); + PyErr_SetString(state->error, "the current screen has no terminal"); + return 0; +} + #define PyCursesStatefulSetupTermCalled(MODULE) \ do { \ if (!_PyCursesStatefulCheckFunction(MODULE, \ @@ -370,7 +399,8 @@ _PyCursesStatefulCheckFunction(PyObject *module, do { \ if (!_PyCursesStatefulCheckFunction(MODULE, \ curses_initscr_called, \ - "initscr")) \ + "initscr") \ + || !_PyCursesStatefulCheckTerminal(MODULE)) \ { \ return 0; \ } \ @@ -763,7 +793,9 @@ curses_getcchar(const cchar_t *wcval, wchar_t *wstr, attr_t *attrs, int *pair) /* getcchar() is not guaranteed to write the text of an empty cell, so make the output an empty string by default. */ wstr[0] = L'\0'; -#if _NCURSES_EXTENDED_COLOR_FUNCS + /* getcchar()'s opts slot returns the extended color pair, but ncurses + returned ERR for a non-NULL opts until 6.3 (patch 20210116). */ +#if _NCURSES_EXTENDED_COLOR_FUNCS && NCURSES_VERSION_PATCH+0 >= 20210116 int rtn = getcchar(wcval, wstr, attrs, &spair, pair); #else int rtn = getcchar(wcval, wstr, attrs, &spair, NULL); @@ -1791,6 +1823,42 @@ curses_window_put_cells(PyCursesWindowObject *self, PyObject *obj, Py_RETURN_NONE; \ } +/* Same as Window_OneArgNoReturnVoidFunction() for a function that needs + a terminal. */ +#define Window_OneArgNoReturnVoidTerminalFunction(X, TYPE, PARSESTR) \ + static PyObject * PyCursesWindow_ ## X \ + (PyObject *op, PyObject *args) \ + { \ + TYPE arg1; \ + if (!PyArg_ParseTuple(args, PARSESTR, &arg1)) { \ + return NULL; \ + } \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + if (!curses_window_check_terminal(self)) { \ + return NULL; \ + } \ + X(self->win, arg1); \ + Py_RETURN_NONE; \ + } + +/* Same as Window_OneArgNoReturnFunction() for a function that needs + a terminal. */ +#define Window_OneArgNoReturnTerminalFunction(X, TYPE, PARSESTR) \ + static PyObject * PyCursesWindow_ ## X \ + (PyObject *op, PyObject *args) \ + { \ + TYPE arg1; \ + if (!PyArg_ParseTuple(args, PARSESTR, &arg1)) { \ + return NULL; \ + } \ + PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); \ + if (!curses_window_check_terminal(self)) { \ + return NULL; \ + } \ + int code = X(self->win, arg1); \ + return curses_window_check_err(self, code, # X, NULL); \ + } + #define Window_OneArgNoReturnFunction(X, TYPE, PARSESTR) \ static PyObject * PyCursesWindow_ ## X \ (PyObject *op, PyObject *args) \ @@ -1891,7 +1959,7 @@ Window_NoArgNoReturnVoidFunction(wclrtoeol) Window_NoArgNoReturnVoidFunction(wclrtobot) Window_NoArgNoReturnVoidFunction(wclear) -Window_OneArgNoReturnVoidFunction(idcok, int, "i;True(1) or False(0)") +Window_OneArgNoReturnVoidTerminalFunction(idcok, int, "i;True(1) or False(0)") #ifdef HAVE_CURSES_IMMEDOK Window_OneArgNoReturnVoidFunction(immedok, int, "i;True(1) or False(0)") #endif @@ -1903,8 +1971,8 @@ Window_NoArg2TupleReturnFunction(getmaxyx, int, "ii") Window_NoArg2TupleReturnFunction(getparyx, int, "ii") Window_OneArgNoReturnFunction(clearok, int, "i;True(1) or False(0)") -Window_OneArgNoReturnFunction(idlok, int, "i;True(1) or False(0)") -Window_OneArgNoReturnFunction(keypad, int, "i;True(1) or False(0)") +Window_OneArgNoReturnTerminalFunction(idlok, int, "i;True(1) or False(0)") +Window_OneArgNoReturnTerminalFunction(keypad, int, "i;True(1) or False(0)") Window_OneArgNoReturnFunction(leaveok, int, "i;True(1) or False(0)") Window_OneArgNoReturnFunction(nodelay, int, "i;True(1) or False(0)") Window_OneArgNoReturnFunction(notimeout, int, "i;True(1) or False(0)") @@ -2409,6 +2477,7 @@ _curses_window_attrset_impl(PyCursesWindowObject *self, attr_t attr) return curses_window_check_err(self, rtn, "wattrset", "attrset"); } +#ifdef HAVE_CURSES_WATTR_GET /*[clinic input] _curses.window.attr_get @@ -2434,7 +2503,9 @@ _curses_window_attr_get_impl(PyCursesWindowObject *self) } return Py_BuildValue("(ki)", (unsigned long)attrs, (int)pair); } +#endif /* HAVE_CURSES_WATTR_GET */ +#ifdef HAVE_CURSES_WATTR_SET /*[clinic input] _curses.window.attr_set @@ -2458,7 +2529,9 @@ _curses_window_attr_set_impl(PyCursesWindowObject *self, attr_t attr, #endif return curses_window_check_err(self, rtn, "wattr_set", "attr_set"); } +#endif /* HAVE_CURSES_WATTR_SET */ +#ifdef HAVE_CURSES_WATTR_ON /*[clinic input] _curses.window.attr_on @@ -2475,7 +2548,9 @@ _curses_window_attr_on_impl(PyCursesWindowObject *self, attr_t attr) int rtn = wattr_on(self->win, attr, NULL); return curses_window_check_err(self, rtn, "wattr_on", "attr_on"); } +#endif /* HAVE_CURSES_WATTR_ON */ +#ifdef HAVE_CURSES_WATTR_OFF /*[clinic input] _curses.window.attr_off @@ -2492,7 +2567,9 @@ _curses_window_attr_off_impl(PyCursesWindowObject *self, attr_t attr) int rtn = wattr_off(self->win, attr, NULL); return curses_window_check_err(self, rtn, "wattr_off", "attr_off"); } +#endif /* HAVE_CURSES_WATTR_OFF */ +#ifdef HAVE_CURSES_WCOLOR_SET /*[clinic input] _curses.window.color_set @@ -2514,6 +2591,7 @@ _curses_window_color_set_impl(PyCursesWindowObject *self, int pair) #endif return curses_window_check_err(self, rtn, "wcolor_set", "color_set"); } +#endif /* HAVE_CURSES_WCOLOR_SET */ /*[clinic input] _curses.window.getattrs @@ -2948,6 +3026,10 @@ _curses_window_echochar_impl(PyCursesWindowObject *self, PyObject *ch, int group_right_1, attr_t attr) /*[clinic end generated code: output=ab03afa580aa6a2a input=cd74c42aadcc7e30]*/ { + if (!curses_window_check_terminal(self)) { + return NULL; + } + chtype ch_; #ifdef HAVE_NCURSESW cchar_t wch; @@ -3194,6 +3276,10 @@ _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, { int rtn; + if (!curses_window_check_terminal(self)) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS if (!group_right_1) { rtn = wgetch(self->win); @@ -3242,6 +3328,10 @@ _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, { int rtn; + if (!curses_window_check_terminal(self)) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS if (!group_right_1) { rtn = wgetch(self->win); @@ -3293,6 +3383,10 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) /*[clinic end generated code: output=9f4f86e91fe50ef3 input=dd7e5367fb49dc48]*/ { + if (!curses_window_check_terminal(self)) { + return NULL; + } + #ifdef HAVE_NCURSESW int ct; wint_t rtn; @@ -3442,6 +3536,10 @@ static PyObject * PyCursesWindow_getstr(PyObject *op, PyObject *args) { PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + + if (!curses_window_check_terminal(self)) { + return NULL; + } return curses_window_getstr_bytes(self, args, "_curses.window.getstr"); } @@ -3544,6 +3642,24 @@ _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, if (type == 0) { return NULL; } + if (type == 1) { + /* winsch() does not locale-decode a byte above 127 on a wide build, + unlike waddch(), so decode it here and insert it as a wide + character. (gh-153864) */ + chtype cch = ch_ & A_CHARTEXT; + if (cch > 127) { + wint_t wc = btowc((int)cch); + if (wc != WEOF) { + wchar_t wstr[2] = { (wchar_t)wc, L'\0' }; + attr_t cattr = (attr_t)((ch_ | attr) & ~(chtype)A_CHARTEXT); + if (curses_setcchar(&wch, wstr, cattr, PAIR_NUMBER(cattr)) == ERR) { + curses_window_set_error(self, "setcchar", "insch"); + return NULL; + } + type = 2; + } + } + } if (type == 2) { if (!group_left_1) { rtn = wins_wch(self->win, &wch); @@ -3596,7 +3712,40 @@ _curses_window_inch_impl(PyCursesWindowObject *self, int group_right_1, { chtype rtn; const char *funcname; - +#ifdef HAVE_NCURSESW + /* ncursesw's winch() returns the character's whole code point instead of + its locale byte, overflowing the chtype's 8-bit character field into the + color and attribute bits; read the wide cell and rebuild it instead. */ + cchar_t cell = {0}; + int rc; + if (!group_right_1) { + rc = win_wch(self->win, &cell); + funcname = "win_wch"; + } + else { + rc = mvwin_wch(self->win, y, x, &cell); + funcname = "mvwin_wch"; + } + if (rc == ERR) { + curses_window_set_error(self, funcname, "inch"); + return NULL; + } + wchar_t wstr[CCHARW_MAX + 1]; + attr_t attrs; + int pair; + if (curses_getcchar(&cell, wstr, &attrs, &pair) == ERR) { + curses_window_set_error(self, "getcchar", "inch"); + return NULL; + } + int byte = 0; + if (wstr[0] != L'\0' && wstr[1] == L'\0') { + byte = wctob(wstr[0]); + if (byte == EOF) { + byte = 0; + } + } + rtn = (chtype)byte | (attrs & ~(attr_t)A_COLOR) | COLOR_PAIR(pair); +#else if (!group_right_1) { rtn = winch(self->win); funcname = "winch"; @@ -3609,6 +3758,7 @@ _curses_window_inch_impl(PyCursesWindowObject *self, int group_right_1, curses_window_set_error(self, funcname, "inch"); return NULL; } +#endif return PyLong_FromUnsignedLong(rtn); } @@ -3691,6 +3841,10 @@ static PyObject * PyCursesWindow_get_wstr(PyObject *op, PyObject *args) { PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + + if (!curses_window_check_terminal(self)) { + return NULL; + } #ifdef HAVE_NCURSESW int rtn, use_xy = 0, y = 0, x = 0; unsigned int max_buf_size = 2048; @@ -4207,6 +4361,10 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self) { int rtn; + if (!curses_window_check_terminal(self)) { + return NULL; + } + #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { @@ -4437,6 +4595,10 @@ _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, { int rtn; + if (!curses_window_check_terminal(self)) { + return NULL; + } + #ifdef py_is_pad if (py_is_pad(self->win)) { if (!group_right_1) { @@ -6033,6 +6195,7 @@ _curses_scr_init(PyObject *module, PyObject *filename) /*[clinic end generated code: output=2e861d381d710419 input=81c45e4702124ef6]*/ ScreenDumpFunctionBody(scr_init) +#ifdef HAVE_CURSES_SCR_SET /*[clinic input] _curses.scr_set @@ -6049,6 +6212,7 @@ static PyObject * _curses_scr_set(PyObject *module, PyObject *filename) /*[clinic end generated code: output=6056fdec12c5935f input=d248c20543cc289b]*/ ScreenDumpFunctionBody(scr_set) +#endif /* HAVE_CURSES_SCR_SET */ #endif /* HAVE_CURSES_SCR_DUMP */ /*[clinic input] @@ -6507,7 +6671,23 @@ _curses_initscr_impl(PyObject *module) _curses_set_null_error(state, "wrefresh", "initscr"); return NULL; } - PyObject *winobj = PyCursesWindow_New(state, stdscr, NULL, NULL, NULL); + if (state->topscreen != NULL) { + /* The current screen is one made by newterm(); return its own + standard window instead of a second wrapper over the same + WINDOW, which would delwin() it on its own. */ + PyCursesScreenObject *so = (PyCursesScreenObject *)state->topscreen; + if (so->stdscr_win != NULL) { + if (curses_update_screen_encoding(so->stdscr_win) < 0) { + return NULL; + } + return Py_NewRef(so->stdscr_win); + } + } + /* Attach the current screen, like newwin(), newpad() and getwin() do, + so that the window keeps its screen alive. It is NULL for the + screen created by initscr(), which has no screen object. */ + PyObject *winobj = PyCursesWindow_New(state, stdscr, NULL, NULL, + state->topscreen); if (winobj == NULL) { return NULL; } @@ -6768,11 +6948,17 @@ _curses_set_term(PyObject *module, PyObject *screen) if (so == NULL) { return NULL; } + cursesmodule_state *state = get_cursesmodule_state(module); + if (so->stdscr_win == NULL) { + /* A screen from new_prescr() has no terminal, so it cannot become the + current one: a later refresh would dereference NULL in curses. */ + PyErr_SetString(state->error, "the screen has no terminal"); + return NULL; + } set_term(so->screen); if (!update_lines_cols(module)) { return NULL; } - cursesmodule_state *state = get_cursesmodule_state(module); PyObject *prev = state->topscreen; /* steal the owned reference */ state->topscreen = Py_NewRef(screen); return prev != NULL ? prev : Py_NewRef(Py_None); @@ -7412,9 +7598,10 @@ _curses_qiflush_impl(PyObject *module, int flag) Py_RETURN_NONE; } -#if defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM) /* Internal helper used for updating curses.LINES, curses.COLS, _curses.LINES - * and _curses.COLS. Returns 1 on success and 0 on failure. */ + * and _curses.COLS. Returns 1 on success and 0 on failure. Used + * unconditionally (e.g. by set_term()), so it must not be gated on resizeterm(). + */ static int update_lines_cols(PyObject *private_module) { @@ -7483,8 +7670,6 @@ _curses_update_lines_cols_impl(PyObject *module) Py_RETURN_NONE; } -#endif - /*[clinic input] _curses.raw @@ -8321,6 +8506,7 @@ _curses_slk_attr_impl(PyObject *module) } #endif +#ifdef HAVE_CURSES_SLK_ATTR_ON /*[clinic input] _curses.slk_attr_on @@ -8338,7 +8524,9 @@ _curses_slk_attr_on_impl(PyObject *module, attr_t attr) return curses_check_err(module, slk_attr_on(attr, NULL), "slk_attr_on", NULL); } +#endif /* HAVE_CURSES_SLK_ATTR_ON */ +#ifdef HAVE_CURSES_SLK_ATTR_OFF /*[clinic input] _curses.slk_attr_off @@ -8356,7 +8544,9 @@ _curses_slk_attr_off_impl(PyObject *module, attr_t attr) return curses_check_err(module, slk_attr_off(attr, NULL), "slk_attr_off", NULL); } +#endif /* HAVE_CURSES_SLK_ATTR_OFF */ +#ifdef HAVE_CURSES_SLK_ATTR_SET /*[clinic input] _curses.slk_attr_set @@ -8380,7 +8570,9 @@ _curses_slk_attr_set_impl(PyObject *module, attr_t attr, int pair) #endif return curses_check_err(module, rtn, "slk_attr_set", NULL); } +#endif /* HAVE_CURSES_SLK_ATTR_SET */ +#ifdef HAVE_CURSES_SLK_COLOR /*[clinic input] _curses.slk_color @@ -8397,6 +8589,7 @@ _curses_slk_color_impl(PyObject *module, int pair) PyCursesStatefulInitialised(module); return curses_check_err(module, slk_color((short)pair), "slk_color", NULL); } +#endif /* HAVE_CURSES_SLK_COLOR */ #ifdef HAVE_CURSES_USE_ENV /*[clinic input] diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index dc1b3c06bed9521..6fbce0ed85e695d 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -1499,6 +1499,20 @@ static int context_clear(PyObject *op) { PyDecContextObject *self = _PyDecContextObject_CAST(op); + /* Since traps and flags hold a borrowed reference to the + flags stored in the context object, these references need + to be cleared when the context object is deallocated + because traps and flags can survive. See gh-146011. */ + PyDecSignalDictObject *traps = _PyDecSignalDictObject_CAST(self->traps); + PyDecSignalDictObject *flags = _PyDecSignalDictObject_CAST(self->flags); + + if (traps != NULL) { + traps->flags = NULL; + } + if (flags != NULL) { + flags->flags = NULL; + } + Py_CLEAR(self->traps); Py_CLEAR(self->flags); return 0; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index b4595c55d519b93..1ab230218124a46 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -382,9 +382,14 @@ partial_vectorcall(PyObject *self, PyObject *const *args, return NULL; } - PyObject **pto_args = _PyTuple_ITEMS(pto->args); - Py_ssize_t pto_nargs = PyTuple_GET_SIZE(pto->args); - Py_ssize_t pto_nkwds = PyDict_GET_SIZE(pto->kw); + PyObject *result = NULL; + PyObject *partial_function = Py_NewRef(pto->fn); + PyObject *partial_args = Py_NewRef(pto->args); + PyObject *partial_keywords = Py_NewRef(pto->kw); + + PyObject **pto_args = _PyTuple_ITEMS(partial_args); + Py_ssize_t pto_nargs = PyTuple_GET_SIZE(partial_args); + Py_ssize_t pto_nkwds = PyDict_GET_SIZE(partial_keywords); Py_ssize_t nkwds = kwnames == NULL ? 0 : PyTuple_GET_SIZE(kwnames); Py_ssize_t nargskw = nargs + nkwds; @@ -392,8 +397,9 @@ partial_vectorcall(PyObject *self, PyObject *const *args, if (!pto_nkwds) { /* Fast path if we're called without arguments */ if (nargskw == 0) { - return _PyObject_VectorcallTstate(tstate, pto->fn, pto_args, - pto_nargs, NULL); + result = _PyObject_VectorcallTstate(tstate, partial_function, pto_args, + pto_nargs, NULL); + goto done; } /* Use PY_VECTORCALL_ARGUMENTS_OFFSET to prepend a single @@ -402,10 +408,10 @@ partial_vectorcall(PyObject *self, PyObject *const *args, PyObject **newargs = (PyObject **)args - 1; PyObject *tmp = newargs[0]; newargs[0] = pto_args[0]; - PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, newargs, - nargs + 1, kwnames); + result = _PyObject_VectorcallTstate(tstate, partial_function, newargs, + nargs + 1, kwnames); newargs[0] = tmp; - return ret; + goto done; } } @@ -435,7 +441,8 @@ partial_vectorcall(PyObject *self, PyObject *const *args, else { stack = PyMem_Malloc(init_stack_size * sizeof(PyObject *)); if (stack == NULL) { - return PyErr_NoMemory(); + PyErr_NoMemory(); + goto done; } } @@ -457,20 +464,20 @@ partial_vectorcall(PyObject *self, PyObject *const *args, for (Py_ssize_t i = 0; i < nkwds; ++i) { key = PyTuple_GET_ITEM(kwnames, i); val = args[nargs + i]; - int contains = PyDict_Contains(pto->kw, key); + int contains = PyDict_Contains(partial_keywords, key); if (contains < 0) { - goto error; + goto clean_stack; } else if (contains == 1) { if (pto_kw_merged == NULL) { - pto_kw_merged = PyDict_Copy(pto->kw); + pto_kw_merged = PyDict_Copy(partial_keywords); if (pto_kw_merged == NULL) { - goto error; + goto clean_stack; } } if (PyDict_SetItem(pto_kw_merged, key, val) < 0) { Py_DECREF(pto_kw_merged); - goto error; + goto clean_stack; } } else { @@ -486,7 +493,7 @@ partial_vectorcall(PyObject *self, PyObject *const *args, tot_kwnames = PyTuple_New(tot_nkwds - n_merges); if (tot_kwnames == NULL) { Py_XDECREF(pto_kw_merged); - goto error; + goto clean_stack; } for (Py_ssize_t i = 0; i < n_tail; ++i) { key = Py_NewRef(stack[tot_nargskw + i]); @@ -496,7 +503,7 @@ partial_vectorcall(PyObject *self, PyObject *const *args, /* Copy pto_keywords with overlapping call keywords merged * Note, tail is already coppied. */ Py_ssize_t pos = 0, i = 0; - PyObject *keyword_dict = n_merges ? pto_kw_merged : pto->kw; + PyObject *keyword_dict = n_merges ? pto_kw_merged : partial_keywords; Py_BEGIN_CRITICAL_SECTION(keyword_dict); while (PyDict_Next(keyword_dict, &pos, &key, &val)) { assert(i < pto_nkwds); @@ -515,10 +522,8 @@ partial_vectorcall(PyObject *self, PyObject *const *args, tmp_stack = PyMem_Realloc(stack, (tot_nargskw - n_merges) * sizeof(PyObject *)); if (tmp_stack == NULL) { Py_DECREF(tot_kwnames); - if (stack != small_stack) { - PyMem_Free(stack); - } - return PyErr_NoMemory(); + PyErr_NoMemory(); + goto clean_stack; } stack = tmp_stack; } @@ -547,21 +552,22 @@ partial_vectorcall(PyObject *self, PyObject *const *args, memcpy(stack + pto_nargs, args, nargs * sizeof(PyObject*)); } - PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, stack, - tot_nargs, tot_kwnames); - if (stack != small_stack) { - PyMem_Free(stack); - } + result = _PyObject_VectorcallTstate(tstate, partial_function, stack, + tot_nargs, tot_kwnames); if (pto_nkwds) { Py_DECREF(tot_kwnames); } - return ret; - error: + clean_stack: if (stack != small_stack) { PyMem_Free(stack); } - return NULL; + + done: + Py_DECREF(partial_function); + Py_DECREF(partial_args); + Py_DECREF(partial_keywords); + return result; } /* Set pto->vectorcall depending on the parameters of the partial object */ diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index 3d14ec3f8f94a92..7d6053d85cd9e4a 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1054,7 +1054,9 @@ bytesio_setstate_lock_held(PyObject *op, PyObject *state) return NULL; } else { - self->dict = Py_NewRef(dict); + /* The LOAD_ATTR specializations read the dict slot lock-free + with an acquire load, so pair it with a release store. */ + FT_ATOMIC_STORE_PTR_RELEASE(self->dict, Py_NewRef(dict)); } } diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index b8601383ad0a26f..27605b4c44239e9 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -407,8 +407,10 @@ _io_StringIO_readline_impl(stringio *self, Py_ssize_t size) } static PyObject * -stringio_iternext(PyObject *op) +stringio_iternext_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + PyObject *line; stringio *self = stringio_CAST(op); @@ -444,6 +446,16 @@ stringio_iternext(PyObject *op) return line; } +static PyObject * +stringio_iternext(PyObject *op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = stringio_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; +} + /*[clinic input] @critical_section _io.StringIO.truncate diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 0f71358a180dd29..20783dc00f90d3c 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -443,6 +443,7 @@ _locale_strxfrm_impl(PyObject *module, PyObject *str) { Py_ssize_t n1; wchar_t *s = NULL, *buf = NULL; + wchar_t dummy[1]; size_t n2; PyObject *result = NULL; @@ -456,7 +457,9 @@ _locale_strxfrm_impl(PyObject *module, PyObject *str) } errno = 0; - n2 = wcsxfrm(NULL, s, 0); + /* Query the size with a real one-element buffer: DragonFly BSD's + wcsxfrm() crashes when the destination is NULL or the size is 0. */ + n2 = wcsxfrm(dummy, s, 1); if (errno && errno != ERANGE) { PyErr_SetFromErrno(PyExc_OSError); goto exit; diff --git a/Modules/_remote_debugging/code_objects.c b/Modules/_remote_debugging/code_objects.c index ab889a130ee4e7e..f83252524b96ff4 100644 --- a/Modules/_remote_debugging/code_objects.c +++ b/Modules/_remote_debugging/code_objects.c @@ -7,6 +7,8 @@ #include "_remote_debugging.h" +#define MAX_LINETABLE_SIZE (64 * 1024) + /* ============================================================================ * TLBC CACHING FUNCTIONS (Py_GIL_DISABLED only) * ============================================================================ */ @@ -186,11 +188,7 @@ parse_linetable(const uintptr_t addrq, const char* linetable, Py_ssize_t linetab int computed_line = firstlineno; // Running accumulator, separate from output int val; // Temporary for varint results uint8_t byte; // Temporary for byte reads - const size_t MAX_LINETABLE_ENTRIES = 65536; - size_t entry_count = 0; - - while (ptr < end && *ptr != '\0' && entry_count < MAX_LINETABLE_ENTRIES) { - entry_count++; + while (ptr < end && *ptr != '\0') { uint8_t first_byte = *(ptr++); uint8_t code = (first_byte >> 3) & 15; size_t length = (first_byte & 7) + 1; @@ -387,7 +385,8 @@ parse_code_object(RemoteUnwinderObject *unwinder, } linetable = read_py_bytes(unwinder, - GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.linetable), 4096); + GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.linetable), + MAX_LINETABLE_SIZE); if (!linetable) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read linetable from code object"); goto error; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 20564e1ba4165a6..9f8a6a58cd9327e 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -3863,6 +3863,10 @@ _ssl__SSLContext_set_client_sigalgs_impl(PySSLContext *self, #ifdef OPENSSL_IS_AWSLC _setSSLError(get_state_ctx(self), "can't set client sigalgs on AWS-LC", 0, __FILE__, __LINE__); return NULL; +#elif defined(LIBRESSL_VERSION_NUMBER) + PyErr_SetString(PyExc_NotImplementedError, + "setting client sigalgs is not supported by LibreSSL"); + return NULL; #else if (!SSL_CTX_set1_client_sigalgs_list(self->ctx, sigalgslist)) { _setSSLError(get_state_ctx(self), "unrecognized signature algorithm", 0, __FILE__, __LINE__); @@ -3884,11 +3888,17 @@ _ssl__SSLContext_set_server_sigalgs_impl(PySSLContext *self, const char *sigalgslist) /*[clinic end generated code: output=31ecb1d310285644 input=653b752e4f8d801b]*/ { +#ifdef LIBRESSL_VERSION_NUMBER + PyErr_SetString(PyExc_NotImplementedError, + "setting server sigalgs is not supported by LibreSSL"); + return NULL; +#else if (!SSL_CTX_set1_sigalgs_list(self->ctx, sigalgslist)) { _setSSLError(get_state_ctx(self), "unrecognized signature algorithm", 0, __FILE__, __LINE__); return NULL; } Py_RETURN_NONE; +#endif } static int diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 799390c27053287..fb18a866e628128 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -15,6 +15,10 @@ #include "frameobject.h" // PyFrame_New() #include "marshal.h" // PyMarshal_WriteLongToFile() +#ifdef bool +# error "The public headers should not include , see gh-90904" +#endif + #include // FLT_MAX #include #include // offsetof() @@ -26,10 +30,6 @@ # include // sysctlbyname() #endif -#ifdef bool -# error "The public headers should not include , see gh-48924" -#endif - #include "_testcapi/util.h" diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ea3ad2b81c28668..e9950bb232431c6 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2312,8 +2312,7 @@ destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) } t2 = PyThreadState_New(interp); prev = PyThreadState_Swap(t2); - PyThreadState_Clear(t1); - PyThreadState_Delete(t1); + // t1 is deliberately left alive; Py_EndInterpreter() must clean it up. Py_EndInterpreter(t2); PyThreadState_Swap(prev); } @@ -3423,6 +3422,9 @@ module_exec(PyObject *module) if (_PyTestInternalCapi_Init_Tuple(module) < 0) { return 1; } + if (_PyTestInternalCapi_Init_TypeCache(module) < 0) { + return 1; + } Py_ssize_t sizeof_gc_head = 0; #ifndef Py_GIL_DISABLED diff --git a/Modules/_testinternalcapi/parts.h b/Modules/_testinternalcapi/parts.h index 81f536c3babb18c..3775792b8b87156 100644 --- a/Modules/_testinternalcapi/parts.h +++ b/Modules/_testinternalcapi/parts.h @@ -16,5 +16,6 @@ int _PyTestInternalCapi_Init_Set(PyObject *module); int _PyTestInternalCapi_Init_Complex(PyObject *module); int _PyTestInternalCapi_Init_CriticalSection(PyObject *module); int _PyTestInternalCapi_Init_Tuple(PyObject *module); +int _PyTestInternalCapi_Init_TypeCache(PyObject *module); #endif // Py_TESTINTERNALCAPI_PARTS_H diff --git a/Modules/_testinternalcapi/typecache.c b/Modules/_testinternalcapi/typecache.c new file mode 100644 index 000000000000000..b5fe0cae353801b --- /dev/null +++ b/Modules/_testinternalcapi/typecache.c @@ -0,0 +1,95 @@ +// Test wrappers for the per-type lookup cache (pycore_typecache.h). +// +// Insertion is exercised indirectly through normal attribute access (which +// calls _PyType_Lookup); only Lookup and Invalidate need direct wrappers. + +#include "parts.h" + +#include "pycore_critical_section.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_stackref.h" // PyStackRef_AsPyObjectSteal() +#include "pycore_typecache.h" // _PyTypeCache_Lookup() + + +static int +require_type(PyObject *obj) +{ + if (!PyType_Check(obj)) { + PyErr_SetString(PyExc_TypeError, "expected a type"); + return -1; + } + return 0; +} + +static PyObject * +intern_name(PyObject *name) +{ + if (!PyUnicode_CheckExact(name)) { + PyErr_SetString(PyExc_TypeError, "name must be a str"); + return NULL; + } + Py_INCREF(name); + PyUnicode_InternInPlace(&name); + return name; +} + +// type_cache_lookup(type, name) -> (cache_hit, value_or_None, version_tag) +static PyObject * +type_cache_lookup(PyObject *Py_UNUSED(self), PyObject *args) +{ + PyObject *type_obj, *name; + if (!PyArg_ParseTuple(args, "OU", &type_obj, &name)) { + return NULL; + } + if (require_type(type_obj) < 0) { + return NULL; + } + name = intern_name(name); + if (name == NULL) { + return NULL; + } + struct _PyTypeCacheLookupResult r = + _PyTypeCache_Lookup((PyTypeObject *)type_obj, name); + Py_DECREF(name); + PyObject *value; + if (PyStackRef_IsNull(r.value)) { + value = Py_NewRef(Py_None); + } + else { + value = PyStackRef_AsPyObjectSteal(r.value); + } + return Py_BuildValue("(iNk)", + r.cache_hit, value, + (unsigned long)r.version_tag); +} + +static PyObject * +type_cache_invalidate(PyObject *Py_UNUSED(self), PyObject *type_obj) +{ + if (require_type(type_obj) < 0) { + return NULL; + } + Py_BEGIN_CRITICAL_SECTION_MUTEX(&_PyInterpreterState_GET()->types.mutex); + _PyTypeCache_Invalidate((PyTypeObject *)type_obj); + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; +} + + +static PyMethodDef test_methods[] = { + {"type_cache_lookup", type_cache_lookup, METH_VARARGS}, + {"type_cache_invalidate", type_cache_invalidate, METH_O}, + {NULL}, +}; + +int +_PyTestInternalCapi_Init_TypeCache(PyObject *m) +{ + if (PyModule_AddFunctions(m, test_methods) < 0) { + return -1; + } + if (PyModule_AddIntMacro(m, _Py_TYPECACHE_MINSIZE) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 8ce704502af0104..0a562ea9c03110b 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -38,6 +38,9 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Bytes(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Capsule(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_Codec(mod) < 0) { return NULL; } diff --git a/Modules/_testlimitedcapi/capsule.c b/Modules/_testlimitedcapi/capsule.c new file mode 100644 index 000000000000000..07b85823f399ba3 --- /dev/null +++ b/Modules/_testlimitedcapi/capsule.c @@ -0,0 +1,63 @@ +#include "parts.h" + +static void +capsule_destructor(PyObject *op) +{ + /* If non-NULL, the name and the pointer are the same allocation. */ + free((char *)PyCapsule_GetName(op)); +} + +static PyObject * +capsule_new(PyObject *self, PyObject *arg) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + char *name_copy = NULL; + if (name != NULL) { + name_copy = strdup(name); + if (name_copy == NULL) { + return PyErr_NoMemory(); + } + } + static const char dummy = 0; + void *pointer = name_copy != NULL ? (void *)name_copy : (void *)&dummy; + PyObject *capsule = PyCapsule_New(pointer, name_copy, capsule_destructor); + if (capsule == NULL) { + free(name_copy); + } + return capsule; +} + +static PyObject * +pycapsule_import(PyObject *self, PyObject *args) +{ + const char *name; + Py_ssize_t size; + int no_block = 0; + if (!PyArg_ParseTuple(args, "z#|i", &name, &size, &no_block)) { + return NULL; + } + void *pointer = PyCapsule_Import(name, no_block); + if (pointer == NULL) { + return NULL; + } + /* Capsules created by capsule_new() store a copy of their name as the + pointer, so a successful import round-trips the name. Only use this + function with such capsules. */ + return PyUnicode_FromString((const char *)pointer); +} + +static PyMethodDef test_methods[] = { + {"capsule_new", capsule_new, METH_O}, + {"PyCapsule_Import", pycapsule_import, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Capsule(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index a11e0edb8311e31..32c1bbc1b71c977 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -25,6 +25,7 @@ int _PyTestLimitedCAPI_Init_Abstract(PyObject *module); int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); +int _PyTestLimitedCAPI_Init_Capsule(PyObject *module); int _PyTestLimitedCAPI_Init_Codec(PyObject *module); int _PyTestLimitedCAPI_Init_Complex(PyObject *module); int _PyTestLimitedCAPI_Init_Dict(PyObject *module); diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index c31a7e8fea56084..3edc29a75b32ce9 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -20,6 +20,9 @@ #ifdef MS_WINDOWS #include +#ifndef RPC_S_OK +#define RPC_S_OK 0L +#endif #endif #ifndef MS_WINDOWS diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 535784adedb24d8..a649d84a7925a04 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1739,7 +1739,19 @@ _winapi_GetVersion_impl(PyObject *module) #pragma warning(disable:4996) { +#ifdef MS_WINDOWS_DESKTOP return GetVersion(); +#else + OSVERSIONINFOW version_info; + ZeroMemory(&version_info, sizeof(version_info)); + version_info.dwOSVersionInfoSize = sizeof(version_info); + if (GetVersionExW(&version_info)) { + return version_info.dwMinorVersion | + (version_info.dwMajorVersion << 8) | + (version_info.dwBuildNumber << 16); + } + return 0; +#endif } #pragma warning(pop) diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index da1b97c0ea5ad8b..70d99b4bf351a21 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -534,6 +534,8 @@ _curses_window_attrset(PyObject *self, PyObject *arg) return return_value; } +#if defined(HAVE_CURSES_WATTR_GET) + PyDoc_STRVAR(_curses_window_attr_get__doc__, "attr_get($self, /)\n" "--\n" @@ -552,6 +554,10 @@ _curses_window_attr_get(PyObject *self, PyObject *Py_UNUSED(ignored)) return _curses_window_attr_get_impl((PyCursesWindowObject *)self); } +#endif /* defined(HAVE_CURSES_WATTR_GET) */ + +#if defined(HAVE_CURSES_WATTR_SET) + PyDoc_STRVAR(_curses_window_attr_set__doc__, "attr_set($self, attr, pair=0, /)\n" "--\n" @@ -591,6 +597,10 @@ _curses_window_attr_set(PyObject *self, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* defined(HAVE_CURSES_WATTR_SET) */ + +#if defined(HAVE_CURSES_WATTR_ON) + PyDoc_STRVAR(_curses_window_attr_on__doc__, "attr_on($self, attr, /)\n" "--\n" @@ -618,6 +628,10 @@ _curses_window_attr_on(PyObject *self, PyObject *arg) return return_value; } +#endif /* defined(HAVE_CURSES_WATTR_ON) */ + +#if defined(HAVE_CURSES_WATTR_OFF) + PyDoc_STRVAR(_curses_window_attr_off__doc__, "attr_off($self, attr, /)\n" "--\n" @@ -645,6 +659,10 @@ _curses_window_attr_off(PyObject *self, PyObject *arg) return return_value; } +#endif /* defined(HAVE_CURSES_WATTR_OFF) */ + +#if defined(HAVE_CURSES_WCOLOR_SET) + PyDoc_STRVAR(_curses_window_color_set__doc__, "color_set($self, pair, /)\n" "--\n" @@ -672,6 +690,8 @@ _curses_window_color_set(PyObject *self, PyObject *arg) return return_value; } +#endif /* defined(HAVE_CURSES_WCOLOR_SET) */ + PyDoc_STRVAR(_curses_window_getattrs__doc__, "getattrs($self, /)\n" "--\n" @@ -3104,7 +3124,7 @@ PyDoc_STRVAR(_curses_scr_init__doc__, #endif /* defined(HAVE_CURSES_SCR_DUMP) */ -#if defined(HAVE_CURSES_SCR_DUMP) +#if defined(HAVE_CURSES_SCR_DUMP) && defined(HAVE_CURSES_SCR_SET) PyDoc_STRVAR(_curses_scr_set__doc__, "scr_set($module, filename, /)\n" @@ -3120,7 +3140,7 @@ PyDoc_STRVAR(_curses_scr_set__doc__, #define _CURSES_SCR_SET_METHODDEF \ {"scr_set", (PyCFunction)_curses_scr_set, METH_O, _curses_scr_set__doc__}, -#endif /* defined(HAVE_CURSES_SCR_DUMP) */ +#endif /* defined(HAVE_CURSES_SCR_DUMP) && defined(HAVE_CURSES_SCR_SET) */ PyDoc_STRVAR(_curses_halfdelay__doc__, "halfdelay($module, tenths, /)\n" @@ -4752,8 +4772,6 @@ _curses_qiflush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#if (defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM)) - PyDoc_STRVAR(_curses_update_lines_cols__doc__, "update_lines_cols($module, /)\n" "--\n" @@ -4774,8 +4792,6 @@ _curses_update_lines_cols(PyObject *module, PyObject *Py_UNUSED(ignored)) return _curses_update_lines_cols_impl(module); } -#endif /* (defined(HAVE_CURSES_RESIZETERM) || defined(HAVE_CURSES_RESIZE_TERM)) */ - PyDoc_STRVAR(_curses_raw__doc__, "raw($module, flag=True, /)\n" "--\n" @@ -5755,6 +5771,8 @@ _curses_slk_attr(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* (defined(NCURSES_EXT_FUNCS) || defined(PDCURSES)) */ +#if defined(HAVE_CURSES_SLK_ATTR_ON) + PyDoc_STRVAR(_curses_slk_attr_on__doc__, "slk_attr_on($module, attr, /)\n" "--\n" @@ -5782,6 +5800,10 @@ _curses_slk_attr_on(PyObject *module, PyObject *arg) return return_value; } +#endif /* defined(HAVE_CURSES_SLK_ATTR_ON) */ + +#if defined(HAVE_CURSES_SLK_ATTR_OFF) + PyDoc_STRVAR(_curses_slk_attr_off__doc__, "slk_attr_off($module, attr, /)\n" "--\n" @@ -5809,6 +5831,10 @@ _curses_slk_attr_off(PyObject *module, PyObject *arg) return return_value; } +#endif /* defined(HAVE_CURSES_SLK_ATTR_OFF) */ + +#if defined(HAVE_CURSES_SLK_ATTR_SET) + PyDoc_STRVAR(_curses_slk_attr_set__doc__, "slk_attr_set($module, attr, pair=0, /)\n" "--\n" @@ -5847,6 +5873,10 @@ _curses_slk_attr_set(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } +#endif /* defined(HAVE_CURSES_SLK_ATTR_SET) */ + +#if defined(HAVE_CURSES_SLK_COLOR) + PyDoc_STRVAR(_curses_slk_color__doc__, "slk_color($module, pair, /)\n" "--\n" @@ -5874,6 +5904,8 @@ _curses_slk_color(PyObject *module, PyObject *arg) return return_value; } +#endif /* defined(HAVE_CURSES_SLK_COLOR) */ + #if defined(HAVE_CURSES_USE_ENV) PyDoc_STRVAR(_curses_use_env__doc__, @@ -6003,6 +6035,26 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored return _curses_has_extended_color_support_impl(module); } +#ifndef _CURSES_WINDOW_ATTR_GET_METHODDEF + #define _CURSES_WINDOW_ATTR_GET_METHODDEF +#endif /* !defined(_CURSES_WINDOW_ATTR_GET_METHODDEF) */ + +#ifndef _CURSES_WINDOW_ATTR_SET_METHODDEF + #define _CURSES_WINDOW_ATTR_SET_METHODDEF +#endif /* !defined(_CURSES_WINDOW_ATTR_SET_METHODDEF) */ + +#ifndef _CURSES_WINDOW_ATTR_ON_METHODDEF + #define _CURSES_WINDOW_ATTR_ON_METHODDEF +#endif /* !defined(_CURSES_WINDOW_ATTR_ON_METHODDEF) */ + +#ifndef _CURSES_WINDOW_ATTR_OFF_METHODDEF + #define _CURSES_WINDOW_ATTR_OFF_METHODDEF +#endif /* !defined(_CURSES_WINDOW_ATTR_OFF_METHODDEF) */ + +#ifndef _CURSES_WINDOW_COLOR_SET_METHODDEF + #define _CURSES_WINDOW_COLOR_SET_METHODDEF +#endif /* !defined(_CURSES_WINDOW_COLOR_SET_METHODDEF) */ + #ifndef _CURSES_WINDOW_ENCLOSE_METHODDEF #define _CURSES_WINDOW_ENCLOSE_METHODDEF #endif /* !defined(_CURSES_WINDOW_ENCLOSE_METHODDEF) */ @@ -6135,10 +6187,6 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #define _CURSES_MOUSEMASK_METHODDEF #endif /* !defined(_CURSES_MOUSEMASK_METHODDEF) */ -#ifndef _CURSES_UPDATE_LINES_COLS_METHODDEF - #define _CURSES_UPDATE_LINES_COLS_METHODDEF -#endif /* !defined(_CURSES_UPDATE_LINES_COLS_METHODDEF) */ - #ifndef _CURSES_RESIZETERM_METHODDEF #define _CURSES_RESIZETERM_METHODDEF #endif /* !defined(_CURSES_RESIZETERM_METHODDEF) */ @@ -6163,6 +6211,22 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #define _CURSES_SLK_ATTR_METHODDEF #endif /* !defined(_CURSES_SLK_ATTR_METHODDEF) */ +#ifndef _CURSES_SLK_ATTR_ON_METHODDEF + #define _CURSES_SLK_ATTR_ON_METHODDEF +#endif /* !defined(_CURSES_SLK_ATTR_ON_METHODDEF) */ + +#ifndef _CURSES_SLK_ATTR_OFF_METHODDEF + #define _CURSES_SLK_ATTR_OFF_METHODDEF +#endif /* !defined(_CURSES_SLK_ATTR_OFF_METHODDEF) */ + +#ifndef _CURSES_SLK_ATTR_SET_METHODDEF + #define _CURSES_SLK_ATTR_SET_METHODDEF +#endif /* !defined(_CURSES_SLK_ATTR_SET_METHODDEF) */ + +#ifndef _CURSES_SLK_COLOR_METHODDEF + #define _CURSES_SLK_COLOR_METHODDEF +#endif /* !defined(_CURSES_SLK_COLOR_METHODDEF) */ + #ifndef _CURSES_USE_ENV_METHODDEF #define _CURSES_USE_ENV_METHODDEF #endif /* !defined(_CURSES_USE_ENV_METHODDEF) */ @@ -6174,4 +6238,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #define _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_ASSUME_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=36fcacafc5044720 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ae1359964feefd64 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index ecb5257cf6b2408..fe40ece030116bf 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -745,7 +745,7 @@ PyDoc_STRVAR(cmath_rect__doc__, #define CMATH_RECT_METHODDEF \ {"rect", _PyCFunction_CAST(cmath_rect), METH_FASTCALL, cmath_rect__doc__}, -static PyObject * +static Py_complex cmath_rect_impl(PyObject *module, double r, double phi); static PyObject * @@ -754,6 +754,7 @@ cmath_rect(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; double r; double phi; + Py_complex _return_value; if (!_PyArg_CheckPositional("rect", nargs, 2, 2)) { goto exit; @@ -778,7 +779,18 @@ cmath_rect(PyObject *module, PyObject *const *args, Py_ssize_t nargs) goto exit; } } - return_value = cmath_rect_impl(module, r, phi); + _return_value = cmath_rect_impl(module, r, phi); + if (errno == EDOM) { + PyErr_SetString(PyExc_ValueError, "math domain error"); + goto exit; + } + else if (errno == ERANGE) { + PyErr_SetString(PyExc_OverflowError, "math range error"); + goto exit; + } + else { + return_value = PyComplex_FromCComplex(_return_value); + } exit: return return_value; @@ -987,4 +999,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=7d5ad4cf258526cd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=89513888cb105a65 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 1f7e414b2dfe48f..ac9b63dec9eb440 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -8801,6 +8801,7 @@ PyDoc_STRVAR(os_pwritev__doc__, "- RWF_APPEND\n" "- RWF_DONTCACHE\n" "- RWF_ATOMIC\n" +"- RWF_NOSIGNAL\n" "\n" "Using non-zero flags requires Linux 4.7 or newer."); @@ -13733,4 +13734,4 @@ os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py #ifndef OS__EMSCRIPTEN_LOG_METHODDEF #define OS__EMSCRIPTEN_LOG_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ -/*[clinic end generated code: output=d81494ceb6ce44ed input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d641f02a97057666 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/readline.c.h b/Modules/clinic/readline.c.h index dc9381e4b976acd..a7df757f3f6c85c 100644 --- a/Modules/clinic/readline.c.h +++ b/Modules/clinic/readline.c.h @@ -366,7 +366,13 @@ readline_get_pre_input_hook_impl(PyObject *module); static PyObject * readline_get_pre_input_hook(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return readline_get_pre_input_hook_impl(module); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = readline_get_pre_input_hook_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; } #endif /* defined(HAVE_RL_PRE_INPUT_HOOK) */ @@ -651,7 +657,13 @@ readline_get_completer_impl(PyObject *module); static PyObject * readline_get_completer(PyObject *module, PyObject *Py_UNUSED(ignored)) { - return readline_get_completer_impl(module); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(module); + return_value = readline_get_completer_impl(module); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(readline_get_history_item__doc__, @@ -823,4 +835,4 @@ readline_redisplay(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef READLINE_CLEAR_HISTORY_METHODDEF #define READLINE_CLEAR_HISTORY_METHODDEF #endif /* !defined(READLINE_CLEAR_HISTORY_METHODDEF) */ -/*[clinic end generated code: output=4bd95070973cd0e2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=acf4e4c35191cf09 input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 7c736f4610bb988..f6e1475b00ecfbb 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -1060,7 +1060,7 @@ static Py_complex rect_special_values[7][7] = { }; /*[clinic input] -cmath.rect +cmath.rect -> Py_complex_protected r: double phi: double @@ -1069,9 +1069,9 @@ cmath.rect Convert from polar coordinates to rectangular coordinates. [clinic start generated code]*/ -static PyObject * +static Py_complex cmath_rect_impl(PyObject *module, double r, double phi) -/*[clinic end generated code: output=385a0690925df2d5 input=24c5646d147efd69]*/ +/*[clinic end generated code: output=74ff3d17585f3388 input=50e60c5d28c834e6]*/ { Py_complex z; errno = 0; @@ -1115,11 +1115,7 @@ cmath_rect_impl(PyObject *module, double r, double phi) z.imag = r * sin(phi); errno = 0; } - - if (errno != 0) - return math_error(); - else - return PyComplex_FromCComplex(z); + return z; } /*[clinic input] diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 72bfab1abaf9cae..c0023c839ca7fe3 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3675,9 +3675,11 @@ static PyObject * count_repr(PyObject *op) { countobject *lz = countobject_CAST(op); - if (lz->long_cnt == NULL) + if (lz->long_cnt == NULL) { + Py_ssize_t cnt = FT_ATOMIC_LOAD_SSIZE_RELAXED(lz->cnt); return PyUnicode_FromFormat("%s(%zd)", - _PyType_Name(Py_TYPE(lz)), lz->cnt); + _PyType_Name(Py_TYPE(lz)), cnt); + } if (PyLong_Check(lz->long_step)) { long step = PyLong_AsLong(lz->long_step); diff --git a/Modules/mathintegermodule.c b/Modules/mathintegermodule.c index 0f660d461e349f8..6d35c825349e451 100644 --- a/Modules/mathintegermodule.c +++ b/Modules/mathintegermodule.c @@ -180,10 +180,9 @@ that the bound `(a - 1)**2 < (n >> s) < (a + 1)**2` is maintained from one iteration to the next. A sketch of the proof of this is given below. In addition to the proof sketch, a formal, computer-verified proof -of correctness (using Lean) of an equivalent recursive algorithm can be found -here: +of correctness (using Lean) of the algorithm can be found here: - https://github.com/mdickinson/snippets/blob/master/proofs/isqrt/src/isqrt.lean + https://github.com/mdickinson/snippets/tree/41ce2d256fef06fb32f24fe7014cfa95173ac5e0/proofs/isqrt Here's Python code equivalent to the C implementation below: diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 64e5372d73d2f29..eaa1850b8aee1ac 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -220,6 +220,12 @@ static const double logpi = 1.144729885849400174143427351353058711647; static double m_acospi(double x) { + if (x >= 0.5) { + /* acos(x) = 2*asin(sqrt((1 - x)/2)). 1 - x is exact here. + Some libms (old fdlibm derivatives) lose precision in acos(x) + near x = 1, while asin() is accurate for small arguments. */ + return 2.0*asin(sqrt((1.0 - x)/2.0))/pi; + } double r = acos(x)/pi; if (isgreater(r, 1.0)) { return 1.0; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index d55ab3d4d56edb0..e521e95d4ded73f 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -969,10 +969,13 @@ mmap_mmap_resize_impl(mmap_object *self, Py_ssize_t new_size) #ifdef UNIX void *newmap; -#ifdef __linux__ +#if defined(__linux__) || defined(__NetBSD__) + // Linux mremap() refuses to grow a shared anonymous mapping, and + // NetBSD mremap() returns a mapping whose grown region is not backed, + // so accessing it crashes. Reject it here in both cases. if (self->fd == -1 && !(self->flags & MAP_PRIVATE) && new_size > self->size) { PyErr_Format(PyExc_ValueError, - "mmap: can't expand a shared anonymous mapping on Linux"); + "mmap: can't expand a shared anonymous mapping"); return NULL; } #endif diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 36debd6fe7aa153..350f56c393d3b97 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -829,9 +829,9 @@ PyOS_AfterFork(void) #ifdef MS_WINDOWS /* defined in fileutils.c */ void _Py_time_t_to_FILE_TIME(time_t, int, FILETIME *); -void _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *, ULONG, - FILE_BASIC_INFO *, FILE_ID_INFO *, - struct _Py_stat_struct *); +void _Py_attribute_data_to_stat(FILE_STANDARD_INFO*, ULONG, + FILE_BASIC_INFO*, FILE_ID_INFO*, + struct _Py_stat_struct*); void _Py_stat_basic_info_to_stat(FILE_STAT_BASIC_INFORMATION *, struct _Py_stat_struct *); #endif @@ -2025,17 +2025,23 @@ win32_wchdir(LPCWSTR path) static void find_data_to_file_info(WIN32_FIND_DATAW *pFileData, - BY_HANDLE_FILE_INFORMATION *info, + FILE_BASIC_INFO* basic_info, + FILE_STANDARD_INFO* standard_info, ULONG *reparse_tag) { - memset(info, 0, sizeof(*info)); - info->dwFileAttributes = pFileData->dwFileAttributes; - info->ftCreationTime = pFileData->ftCreationTime; - info->ftLastAccessTime = pFileData->ftLastAccessTime; - info->ftLastWriteTime = pFileData->ftLastWriteTime; - info->nFileSizeHigh = pFileData->nFileSizeHigh; - info->nFileSizeLow = pFileData->nFileSizeLow; -/* info->nNumberOfLinks = 1; */ + memset(basic_info, 0, sizeof(*basic_info)); + memset(standard_info, 0, sizeof(*standard_info)); + + basic_info->FileAttributes = pFileData->dwFileAttributes; + basic_info->CreationTime.HighPart = pFileData->ftCreationTime.dwHighDateTime; + basic_info->CreationTime.LowPart = pFileData->ftCreationTime.dwLowDateTime; + basic_info->LastAccessTime.HighPart = pFileData->ftLastAccessTime.dwHighDateTime; + basic_info->LastAccessTime.LowPart = pFileData->ftLastAccessTime.dwLowDateTime; + basic_info->LastWriteTime.HighPart = pFileData->ftLastWriteTime.dwHighDateTime; + basic_info->LastWriteTime.LowPart = pFileData->ftLastWriteTime.dwLowDateTime; + standard_info->EndOfFile.HighPart = pFileData->nFileSizeHigh; + standard_info->EndOfFile.LowPart = pFileData->nFileSizeLow; + if (pFileData->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) *reparse_tag = pFileData->dwReserved0; else @@ -2043,7 +2049,7 @@ find_data_to_file_info(WIN32_FIND_DATAW *pFileData, } static BOOL -attributes_from_dir(LPCWSTR pszFile, BY_HANDLE_FILE_INFORMATION *info, ULONG *reparse_tag) +attributes_from_dir(LPCWSTR pszFile, FILE_BASIC_INFO* basic_info, FILE_STANDARD_INFO* standard_info, ULONG* reparse_tag) { HANDLE hFindFile; WIN32_FIND_DATAW FileData; @@ -2074,7 +2080,7 @@ attributes_from_dir(LPCWSTR pszFile, BY_HANDLE_FILE_INFORMATION *info, ULONG *re return FALSE; } FindClose(hFindFile); - find_data_to_file_info(&FileData, info, reparse_tag); + find_data_to_file_info(&FileData, basic_info, standard_info, reparse_tag); return TRUE; } @@ -2108,10 +2114,9 @@ win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, BOOL traverse) { HANDLE hFile; - BY_HANDLE_FILE_INFORMATION fileInfo; - FILE_BASIC_INFO basicInfo; - FILE_BASIC_INFO *pBasicInfo = NULL; - FILE_ID_INFO idInfo; + FILE_STANDARD_INFO standardInfo = {0}; + FILE_BASIC_INFO basicInfo = {0}; + FILE_ID_INFO idInfo = {0}; FILE_ID_INFO *pIdInfo = NULL; FILE_ATTRIBUTE_TAG_INFO tagInfo = { 0 }; DWORD fileType, error; @@ -2132,7 +2137,7 @@ win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, case ERROR_ACCESS_DENIED: /* Cannot sync or read attributes. */ case ERROR_SHARING_VIOLATION: /* It's a paging file. */ /* Try reading the parent directory. */ - if (!attributes_from_dir(path, &fileInfo, &tagInfo.ReparseTag)) { + if (!attributes_from_dir(path, &basicInfo, &standardInfo, &tagInfo.ReparseTag)) { /* Cannot read the parent directory. */ switch (GetLastError()) { case ERROR_FILE_NOT_FOUND: /* File cannot be found */ @@ -2147,7 +2152,7 @@ win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, return -1; } - if (fileInfo.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { + if (basicInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { if (traverse || !IsReparseTagNameSurrogate(tagInfo.ReparseTag)) { /* The stat call has to traverse but cannot, so fail. */ @@ -2247,9 +2252,8 @@ win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, } } - if (!GetFileInformationByHandle(hFile, &fileInfo) || - !GetFileInformationByHandleEx(hFile, FileBasicInfo, - &basicInfo, sizeof(basicInfo))) { + if (!GetFileInformationByHandleEx(hFile, FileStandardInfo, &standardInfo, sizeof(standardInfo)) || + !GetFileInformationByHandleEx(hFile, FileBasicInfo, &basicInfo, sizeof(basicInfo))) { switch (GetLastError()) { case ERROR_INVALID_PARAMETER: case ERROR_INVALID_FUNCTION: @@ -2264,17 +2268,14 @@ win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, goto cleanup; } - /* Successfully got FileBasicInfo, so we'll pass it along */ - pBasicInfo = &basicInfo; - if (GetFileInformationByHandleEx(hFile, FileIdInfo, &idInfo, sizeof(idInfo))) { /* Successfully got FileIdInfo, so pass it along */ pIdInfo = &idInfo; } } - _Py_attribute_data_to_stat(&fileInfo, tagInfo.ReparseTag, pBasicInfo, pIdInfo, result); - update_st_mode_from_path(path, fileInfo.dwFileAttributes, result); + _Py_attribute_data_to_stat(&standardInfo, tagInfo.ReparseTag, &basicInfo, pIdInfo, result); + update_st_mode_from_path(path, basicInfo.FileAttributes, result); cleanup: if (hFile != INVALID_HANDLE_VALUE) { @@ -9186,7 +9187,9 @@ os_posix_openpt_impl(PyObject *module, int oflag) { int fd; -#if defined(O_CLOEXEC) + // OpenBSD posix_openpt() rejects any flag other than O_RDWR and + // O_NOCTTY; the fd is made non-inheritable below in any case. +#if defined(O_CLOEXEC) && !defined(__OpenBSD__) oflag |= O_CLOEXEC; #endif @@ -9429,11 +9432,30 @@ os_openpty_impl(PyObject *module) goto posix_error; #if defined(HAVE_STROPTS_H) && !defined(HAVE_DEV_PTC) + // Pushing "ptem" makes the slave a terminal, which a session leader + // without a controlling terminal then acquires as one despite O_NOCTTY. + // Note whether we already had one, so a new one can be disowned below. + int had_ctty = 0; +#ifdef TIOCNOTTY + int tty_fd = open("/dev/tty", O_RDONLY | O_NOCTTY); + if (tty_fd >= 0) { + had_ctty = 1; + close(tty_fd); + } +#endif ioctl(slave_fd, I_PUSH, "ptem"); /* push ptem */ ioctl(slave_fd, I_PUSH, "ldterm"); /* push ldterm */ #ifndef __hpux ioctl(slave_fd, I_PUSH, "ttcompat"); /* push ttcompat */ #endif /* __hpux */ +#ifdef TIOCNOTTY + if (!had_ctty && getsid(0) == getpid()) { + // Disown it; TIOCNOTTY sends SIGHUP to the session leader. + PyOS_sighandler_t sig_saved = PyOS_setsig(SIGHUP, SIG_IGN); + ioctl(slave_fd, TIOCNOTTY); + PyOS_setsig(SIGHUP, sig_saved); + } +#endif #endif /* defined(HAVE_STROPTS_H) && !defined(HAVE_DEV_PTC) */ #endif /* HAVE_OPENPTY */ @@ -12941,6 +12963,7 @@ following flags: - RWF_APPEND - RWF_DONTCACHE - RWF_ATOMIC +- RWF_NOSIGNAL Using non-zero flags requires Linux 4.7 or newer. [clinic start generated code]*/ @@ -12948,7 +12971,7 @@ Using non-zero flags requires Linux 4.7 or newer. static Py_ssize_t os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, int flags) -/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=b2e352a22f030e9a]*/ +/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=c202f24f01fa66c9]*/ { Py_ssize_t cnt; Py_ssize_t result; @@ -13486,6 +13509,10 @@ os_posix_fallocate_impl(PyObject *module, int fd, Py_off_t offset, Py_BEGIN_ALLOW_THREADS result = posix_fallocate(fd, offset, length); Py_END_ALLOW_THREADS + // DragonFly BSD returns -1 and sets errno. + if (result == -1) { + result = errno; + } } while (result == EINTR && !(async_err = PyErr_CheckSignals())); if (result == 0) @@ -13533,6 +13560,10 @@ os_posix_fadvise_impl(PyObject *module, int fd, Py_off_t offset, Py_BEGIN_ALLOW_THREADS result = posix_fadvise(fd, offset, length, advice); Py_END_ALLOW_THREADS + // DragonFly BSD returns -1 and sets errno. + if (result == -1) { + result = errno; + } } while (result == EINTR && !(async_err = PyErr_CheckSignals())); if (result == 0) @@ -16003,7 +16034,9 @@ os_cpu_count_impl(PyObject *module) # ifdef MS_WINDOWS_DESKTOP ncpu = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); # else - ncpu = 0; + SYSTEM_INFO sysinfo; + GetSystemInfo(&sysinfo); + ncpu = sysinfo.dwNumberOfProcessors; # endif #elif defined(__hpux) @@ -16670,7 +16703,8 @@ static PyObject * DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) { DirEntry *entry; - BY_HANDLE_FILE_INFORMATION file_info; + FILE_BASIC_INFO basic_info; + FILE_STANDARD_INFO standard_info; ULONG reparse_tag; wchar_t *joined_path; @@ -16708,8 +16742,8 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) goto error; } - find_data_to_file_info(dataW, &file_info, &reparse_tag); - _Py_attribute_data_to_stat(&file_info, reparse_tag, NULL, NULL, &entry->win32_lstat); + find_data_to_file_info(dataW, &basic_info, &standard_info, &reparse_tag); + _Py_attribute_data_to_stat(&standard_info, reparse_tag, &basic_info, NULL, &entry->win32_lstat); /* ctime is only deprecated from 3.12, so we copy birthtime across */ entry->win32_lstat.st_ctime = entry->win32_lstat.st_birthtime; @@ -18244,6 +18278,9 @@ all_ins(PyObject *m) #ifdef RWF_APPEND if (PyModule_AddIntConstant(m, "RWF_APPEND", RWF_APPEND)) return -1; #endif +#ifdef RWF_NOSIGNAL + if (PyModule_AddIntConstant(m, "RWF_NOSIGNAL", RWF_NOSIGNAL)) return -1; +#endif /* constants for splice */ #if defined(HAVE_SPLICE) && defined(__linux__) diff --git a/Modules/readline.c b/Modules/readline.c index 4c965e081c9d1a7..fc79a5866dfd38c 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -578,6 +578,7 @@ readline_set_pre_input_hook_impl(PyObject *module, PyObject *function) /* Get pre-input hook */ /*[clinic input] +@critical_section readline.get_pre_input_hook Get the current pre-input hook function. @@ -585,7 +586,7 @@ Get the current pre-input hook function. static PyObject * readline_get_pre_input_hook_impl(PyObject *module) -/*[clinic end generated code: output=ad56b77a8e8981ca input=fb1e1b1fbd94e4e5]*/ +/*[clinic end generated code: output=ad56b77a8e8981ca input=fbbf0106bd015414]*/ { readlinestate *state = get_readline_state(module); if (state->pre_input_hook == NULL) { @@ -886,6 +887,7 @@ readline_set_completer_impl(PyObject *module, PyObject *function) } /*[clinic input] +@critical_section readline.get_completer Get the current completer function. @@ -893,7 +895,7 @@ Get the current completer function. static PyObject * readline_get_completer_impl(PyObject *module) -/*[clinic end generated code: output=6e6bbd8226d14475 input=6457522e56d70d13]*/ +/*[clinic end generated code: output=6e6bbd8226d14475 input=0df9ba4107115c44]*/ { readlinestate *state = get_readline_state(module); if (state->completer == NULL) { diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 084c2dbcff066e5..73ae1c942daba48 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -745,6 +745,16 @@ set_error(void) } +#if defined(HAVE_HSTRERROR) || defined(HAVE_GAI_STRERROR) +/* Decode a locale-encoded error message from the C library. + It can be localized and use a non-UTF-8 encoding. */ +static PyObject * +decode_error_message(const char *str) +{ + return PyUnicode_DecodeLocale(str, "surrogateescape"); +} +#endif + #if defined(HAVE_GETHOSTBYNAME_R) || defined (HAVE_GETHOSTBYNAME) || defined (HAVE_GETHOSTBYADDR) static PyObject * set_herror(socket_state *state, int h_error) @@ -752,7 +762,7 @@ set_herror(socket_state *state, int h_error) PyObject *v; #ifdef HAVE_HSTRERROR - v = Py_BuildValue("(is)", h_error, hstrerror(h_error)); + v = Py_BuildValue("(iN)", h_error, decode_error_message(hstrerror(h_error))); #else v = Py_BuildValue("(is)", h_error, "host not found"); #endif @@ -779,7 +789,7 @@ set_gaierror(socket_state *state, int error) #endif #ifdef HAVE_GAI_STRERROR - v = Py_BuildValue("(is)", error, gai_strerror(error)); + v = Py_BuildValue("(iN)", error, decode_error_message(gai_strerror(error))); #else v = Py_BuildValue("(is)", error, "getaddrinfo failed"); #endif diff --git a/Modules/timemodule.c b/Modules/timemodule.c index d90bf1f2ef90ed9..70d7e1b3713687a 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -763,6 +763,13 @@ Other codes may be available on your platform. See documentation for\n\ the C library strftime function.\n" #ifdef HAVE_STRFTIME +// gh-154460: OpenBSD's wcsftime() computes %V incorrectly: it returns 53 +// whenever the ISO 8601 week belongs to a different year than tm_year. +// strftime() is not affected. +#ifdef __OpenBSD__ +# undef HAVE_WCSFTIME +#endif + #ifdef HAVE_WCSFTIME #define time_char wchar_t #define format_time wcsftime diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 696ddad8efaae55..d009877dc09fac0 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -213,6 +213,9 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); PyByteArrayObject *obj = ((PyByteArrayObject *)self); + + assert(obj->ob_bytes_object != NULL); + /* All computations are done unsigned to avoid integer overflows (see issue #22335). */ size_t alloc = (size_t) obj->ob_alloc; @@ -236,6 +239,14 @@ bytearray_resize_lock_held(PyObject *self, Py_ssize_t requested_size) return -1; } + /* Resize to 0 resets to empty bytes (see issue #153419). */ + if (requested_size == 0) { + Py_SETREF(obj->ob_bytes_object, + Py_GetConstant(Py_CONSTANT_EMPTY_BYTES)); + bytearray_reinit_from_bytes(obj, 0, 0); + return 0; + } + if (size + logical_offset <= alloc) { /* Current buffer is large enough to host the requested size, decide on a strategy. */ @@ -902,6 +913,20 @@ bytearray_ass_subscript(PyObject *op, PyObject *index, PyObject *values) return ret; } +static PyObject * +bytearray_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *op = PyType_GenericNew(type, args, kwds); + if (op == NULL) { + return NULL; + } + PyByteArrayObject *self = _PyByteArray_CAST(op); + self->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + bytearray_reinit_from_bytes(self, 0, 0); + self->ob_exports = 0; + return op; +} + /*[clinic input] bytearray.__init__ @@ -920,20 +945,16 @@ bytearray___init___impl(PyByteArrayObject *self, PyObject *arg, PyObject *it; PyObject *(*iternext)(PyObject *); - /* First __init__; set ob_bytes_object so ob_bytes is always non-null. */ - if (self->ob_bytes_object == NULL) { - self->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); - bytearray_reinit_from_bytes(self, 0, 0); - self->ob_exports = 0; + /* Disallow any __init__ call if the object is not resizable (has exports) + to make the handling of non-null `source` init values simpler. */ + if (!_canresize(self)) { + return -1; } - if (Py_SIZE(self) != 0) { - /* Empty previous contents (yes, do this first of all!) */ - if (PyByteArray_Resize((PyObject *)self, 0) < 0) - return -1; + /* Empty any previous contents (do this first of all!). */ + if (PyByteArray_Resize((PyObject *)self, 0) < 0) { + return -1; } - - /* Should be caused by first init or the resize to 0. */ assert(self->ob_bytes_object == Py_GetConstantBorrowed(Py_CONSTANT_EMPTY_BYTES)); assert(self->ob_exports == 0); @@ -1555,6 +1576,8 @@ bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n) if (to_take == -1 && PyErr_Occurred()) { return NULL; } + // n.__index__() may have resized self; use the current size. + size = Py_SIZE(self); if (to_take < 0) { to_take += size; } @@ -1607,6 +1630,9 @@ bytearray_take_bytes_impl(PyByteArrayObject *self, PyObject *n) } if (_PyBytes_Resize(&self->ob_bytes_object, to_take) == -1) { + assert(self->ob_bytes_object == NULL); + self->ob_bytes_object = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + bytearray_reinit_from_bytes(self, 0, 0); Py_DECREF(remaining); return NULL; } @@ -2937,7 +2963,7 @@ PyTypeObject PyByteArray_Type = { 0, /* tp_dictoffset */ bytearray___init__, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + bytearray_new, /* tp_new */ PyObject_Free, /* tp_free */ .tp_version_tag = _Py_TYPE_VERSION_BYTEARRAY, }; diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index f63185e14284b1a..ef35dad82e8aaea 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -3380,6 +3380,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) Py_DECREF(v); return (*pv == NULL) ? -1 : 0; } + assert(v != bytes_get_empty()); #ifdef Py_TRACE_REFS _Py_ForgetReference(v); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 03036020b1cb1ae..d7955cc7390a7ab 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -748,6 +748,7 @@ _PyCode_New(struct _PyCodeConstructor *con) #endif if (init_code(co, con) < 0) { + Py_XDECREF(replacement_locations); Py_DECREF(co); return NULL; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 246ffe6c18e9b51..c650aa456d2cc9d 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -7722,7 +7722,7 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) PyDictValues *values = _PyObject_InlineValues(obj); if (FT_ATOMIC_LOAD_UINT8(values->valid)) { PyDictKeysObject *keys = CACHED_KEYS(tp); - for (Py_ssize_t i = 0; i < keys->dk_nentries; i++) { + for (Py_ssize_t i = 0; i < LOAD_KEYS_NENTRIES(keys); i++) { if (FT_ATOMIC_LOAD_PTR_RELAXED(values->values[i]) != NULL) { return 0; } diff --git a/Objects/enumobject.c b/Objects/enumobject.c index fc53f1bfee8dde4..68aa594c5540cee 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -277,10 +277,13 @@ enum_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) enumobject *en = _enumobject_CAST(op); PyObject *result; Py_BEGIN_CRITICAL_SECTION(en); - if (en->en_longindex != NULL) + if (en->en_longindex != NULL) { result = Py_BuildValue("O(OO)", Py_TYPE(en), en->en_sit, en->en_longindex); - else - result = Py_BuildValue("O(On)", Py_TYPE(en), en->en_sit, en->en_index); + } + else { + Py_ssize_t en_index = FT_ATOMIC_LOAD_SSIZE_RELAXED(en->en_index); + result = Py_BuildValue("O(On)", Py_TYPE(en), en->en_sit, en_index); + } Py_END_CRITICAL_SECTION(); return result; } diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 149595e64cec144..fb546ad2673576d 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -11,9 +11,11 @@ #include "pycore_exceptions.h" // struct _Py_exc_state #include "pycore_initconfig.h" #include "pycore_modsupport.h" // _PyArg_NoKeywords() +#include "pycore_moduleobject.h" // _PyModule_CAST() #include "pycore_object.h" #include "pycore_pyerrors.h" // struct _PyErr_SetRaisedException #include "pycore_tuple.h" // _PyTuple_FromPair +#include "pycore_unicodeobject.h" // _PyUnicode_Equal() #include "osdefs.h" // SEP #include "clinic/exceptions.c.h" @@ -2702,6 +2704,65 @@ AttributeError_dealloc(PyObject *self) Py_TYPE(self)->tp_free(self); } +static PyObject * +AttributeError_str(PyObject *op) +{ + PyAttributeErrorObject *self = PyAttributeErrorObject_CAST(op); + PyObject *arg; // borrowed ref + PyObject *obj = NULL, *name = NULL; + + /* .name and .obj are set automatically when attribute lookup fails, so + synthesize a more informative message from them when the caller + didn't supply a meaningful one of their own -- that is, when args is + empty, or contains only the attribute name. Otherwise, use the + message the caller gave. */ + + Py_BEGIN_CRITICAL_SECTION(self); + if ( + self->obj && self->name && PyUnicode_Check(self->name) + && ((PyTuple_GET_SIZE(self->args) == 1 + && PyUnicode_Check(arg = PyTuple_GET_ITEM(self->args, 0)) + && _PyUnicode_Equal(arg, self->name)) + || PyTuple_GET_SIZE(self->args) == 0) + ) { + obj = Py_NewRef(self->obj); + name = Py_NewRef(self->name); + } + Py_END_CRITICAL_SECTION(); + + if (!obj) { + assert(!name); + return BaseException_str(op); /* re-acquires lock */ + } + + PyObject *result = NULL; + if (PyModule_Check(obj)) { + PyModuleObject *mod = _PyModule_CAST(obj); + PyObject *modname; + if (PyDict_GetItemRef(mod->md_dict, &_Py_ID(__name__), &modname) < 0) { + goto done; + } + if (modname && PyUnicode_Check(modname)) { + result = PyUnicode_FromFormat("module %R has no attribute %R", + modname, name); + Py_DECREF(modname); + } else { + Py_XDECREF(modname); + result = PyUnicode_FromFormat("module has no attribute %R", name); + } + } else if (PyType_Check(obj)) { + result = PyUnicode_FromFormat("type object '%N' has no attribute %R", + obj, name); + } else { + result = PyUnicode_FromFormat("'%T' object has no attribute %R", + obj, name); + } +done: + Py_DECREF(obj); + Py_DECREF(name); + return result; +} + static int AttributeError_traverse(PyObject *op, visitproc visit, void *arg) { @@ -2770,7 +2831,7 @@ static PyMethodDef AttributeError_methods[] = { ComplexExtendsException(PyExc_Exception, AttributeError, AttributeError, 0, AttributeError_methods, AttributeError_members, - 0, BaseException_str, 0, "Attribute not found."); + 0, AttributeError_str, 0, "Attribute not found."); /* * SyntaxError extends Exception diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 4e85927def7ea78..348c7dd6967a397 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -186,20 +186,23 @@ PyObject * _Py_make_parameters(PyObject *args) { assert(PyTuple_Check(args) || PyList_Check(args)); + if (Py_EnterRecursiveCall(" in __parameter__ calculation")) { + return NULL; + } + const bool is_args_list = PyList_Check(args); PyObject *tuple_args = NULL; if (is_args_list) { args = tuple_args = PySequence_Tuple(args); if (args == NULL) { - return NULL; + goto cleanup; } } Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t len = nargs; PyObject *parameters = PyTuple_New(len); if (parameters == NULL) { - Py_XDECREF(tuple_args); - return NULL; + goto error; } Py_ssize_t iparam = 0; for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { @@ -210,9 +213,7 @@ _Py_make_parameters(PyObject *args) } int rc = PyObject_HasAttrWithError(t, &_Py_ID(__typing_subst__)); if (rc < 0) { - Py_DECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; + goto error; } if (rc) { iparam += tuple_add(parameters, iparam, t); @@ -221,18 +222,14 @@ _Py_make_parameters(PyObject *args) PyObject *subparams; if (PyObject_GetOptionalAttr(t, &_Py_ID(__parameters__), &subparams) < 0) { - Py_DECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; + goto error; } if (!subparams && (PyTuple_Check(t) || PyList_Check(t))) { // Recursively call _Py_make_parameters for lists/tuples and // add the results to the current parameters. subparams = _Py_make_parameters(t); if (subparams == NULL) { - Py_DECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; + goto error; } } if (subparams && PyTuple_Check(subparams)) { @@ -243,7 +240,7 @@ _Py_make_parameters(PyObject *args) if (_PyTuple_Resize(¶meters, len) < 0) { Py_DECREF(subparams); Py_XDECREF(tuple_args); - return NULL; + goto cleanup; } } for (Py_ssize_t j = 0; j < len2; j++) { @@ -256,13 +253,19 @@ _Py_make_parameters(PyObject *args) } if (iparam < len) { if (_PyTuple_Resize(¶meters, iparam) < 0) { - Py_XDECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; + goto error; } } Py_XDECREF(tuple_args); + Py_LeaveRecursiveCall(); return parameters; + +error: + Py_XDECREF(parameters); + Py_XDECREF(tuple_args); +cleanup: + Py_LeaveRecursiveCall(); + return NULL; } /* If obj is a generic alias, substitute type variables params @@ -939,17 +942,23 @@ static PyObject * ga_iternext(PyObject *op) { gaiterobject *gi = (gaiterobject*)op; - if (gi->obj == NULL) { +#ifdef Py_GIL_DISABLED + PyObject *obj = _Py_atomic_exchange_ptr(&gi->obj, NULL); +#else + PyObject* obj = gi->obj; + gi->obj = NULL; +#endif + if (obj == NULL) { PyErr_SetNone(PyExc_StopIteration); return NULL; } - gaobject *alias = (gaobject *)gi->obj; + gaobject *alias = (gaobject *)obj; PyObject *starred_alias = Py_GenericAlias(alias->origin, alias->args); + Py_DECREF(obj); if (starred_alias == NULL) { return NULL; } ((gaobject *)starred_alias)->starred = true; - Py_SETREF(gi->obj, NULL); return starred_alias; } diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index d92c7daff15dbf3..f6ffa337c632a0b 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1485,9 +1485,11 @@ memoryview_cast_impl(PyMemoryViewObject *self, PyObject *format, CHECK_RESTRICTED(self); if (!MV_C_CONTIGUOUS(self->flags)) { - PyErr_SetString(PyExc_TypeError, - "memoryview: casts are restricted to C-contiguous views"); - return NULL; + if (shape || !MV_F_CONTIGUOUS(self->flags)) { + PyErr_SetString(PyExc_TypeError, + "memoryview: casts are restricted to contiguous views"); + return NULL; + } } if ((shape || self->view.ndim != 1) && zero_in_shape(self)) { PyErr_SetString(PyExc_TypeError, diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index f447403ef31b43a..b8cd6025c20ba56 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1299,8 +1299,6 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) return result; } -// Check if `name` is a lazily pending submodule of module `m`. -// Returns a new reference on success, or NULL with no error set. static PyObject * try_load_lazy_submodule(PyModuleObject *m, PyObject *name) { @@ -1313,10 +1311,13 @@ try_load_lazy_submodule(PyModuleObject *m, PyObject *name) Py_DECREF(mod_name); return NULL; } - PyObject *result = _PyImport_TryLoadLazySubmodule(mod_name, name); + PyObject *result = NULL; + _PyLazySubmoduleImportResult status = + _PyImport_TryLoadLazySubmodule(mod_name, name, &result); Py_DECREF(mod_name); - if (result == NULL) { - PyErr_Clear(); + if (status != _Py_LAZY_SUBMODULE_LOADED) { + assert(status == _Py_LAZY_SUBMODULE_ERROR || + status == _Py_LAZY_SUBMODULE_NOT_FOUND); return NULL; } if (PyDict_SetItem(m->md_dict, name, result) < 0) { diff --git a/Objects/object.c b/Objects/object.c index bd23c2e23881949..fadd9273a36607c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -3292,13 +3292,20 @@ _Py_Dealloc(PyObject *op) PyTypeObject *type = Py_TYPE(op); unsigned long gc_flag = type->tp_flags & Py_TPFLAGS_HAVE_GC; destructor dealloc = type->tp_dealloc; - PyThreadState *tstate = _PyThreadState_GET(); - intptr_t margin = _Py_RecursionLimit_GetMargin(tstate); - if (margin < 2 && gc_flag) { - _PyTrash_thread_deposit_object(tstate, (PyObject *)op); - return; + PyThreadState *tstate = NULL; + intptr_t margin = 0; + if (gc_flag) { + tstate = _PyThreadState_GET(); + margin = _Py_RecursionLimit_GetMargin(tstate); + if (margin < 2) { + _PyTrash_thread_deposit_object(tstate, (PyObject *)op); + return; + } } #ifdef Py_DEBUG + if (tstate == NULL) { + tstate = _PyThreadState_GET(); + } #if !defined(Py_GIL_DISABLED) && !defined(Py_STACKREF_DEBUG) /* This assertion doesn't hold for the free-threading build, as * PyStackRef_CLOSE_SPECIALIZED is not implemented */ @@ -3340,7 +3347,7 @@ _Py_Dealloc(PyObject *op) Py_XDECREF(old_exc); Py_DECREF(type); #endif - if (tstate->delete_later && margin >= 4 && gc_flag) { + if (gc_flag && tstate->delete_later && margin >= 4) { _PyTrash_thread_destroy_chain(tstate); } } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index ffa20ccaf3dfb87..60e26eab069b094 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -21,7 +21,8 @@ #include "pycore_slots.h" // _PySlotIterator_Init #include "pycore_symtable.h" // _Py_Mangle() #include "pycore_tuple.h" // _PyTuple_FromPair -#include "pycore_typeobject.h" // struct type_cache +#include "pycore_typecache.h" // _PyTypeCache_Lookup() +#include "pycore_typeobject.h" // _PyTypes_InitTypes() #include "pycore_unicodeobject.h" // _PyUnicode_Copy #include "pycore_unionobject.h" // _Py_union_type_or #include "pycore_weakref.h" // _PyWeakref_GET_REF() @@ -41,21 +42,7 @@ class object "PyObject *" "&PyBaseObject_Type" /* Support type attribute lookup cache */ -/* The cache can keep references to the names alive for longer than - they normally would. This is why the maximum size is limited to - MCACHE_MAX_ATTR_SIZE, since it might be a problem if very large - strings are used as attribute names. */ -#define MCACHE_MAX_ATTR_SIZE 100 -#define MCACHE_HASH(version, name_hash) \ - (((unsigned int)(version) ^ (unsigned int)(name_hash)) \ - & ((1 << MCACHE_SIZE_EXP) - 1)) - -#define MCACHE_HASH_METHOD(type, name) \ - MCACHE_HASH(FT_ATOMIC_LOAD_UINT_RELAXED((type)->tp_version_tag), \ - ((Py_ssize_t)(name)) >> 3) -#define MCACHE_CACHEABLE_NAME(name) \ - (PyUnicode_CheckExact(name) && \ - (PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE)) +#define MCACHE_CACHEABLE_NAME(name) (PyUnicode_CheckExact(name) && PyUnicode_CHECK_INTERNED(name)) #define NEXT_VERSION_TAG(interp) \ (interp)->types.next_version_tag @@ -969,75 +956,17 @@ _PyType_GetTextSignatureFromInternalDoc(const char *name, const char *internal_d } -static struct type_cache* -get_type_cache(void) -{ - PyInterpreterState *interp = _PyInterpreterState_GET(); - return &interp->types.type_cache; -} - - -static void -type_cache_clear(struct type_cache *cache, PyObject *value) -{ - for (Py_ssize_t i = 0; i < (1 << MCACHE_SIZE_EXP); i++) { - struct type_cache_entry *entry = &cache->hashtable[i]; -#ifdef Py_GIL_DISABLED - _PySeqLock_LockWrite(&entry->sequence); -#endif - entry->version = 0; - Py_XSETREF(entry->name, _Py_XNewRef(value)); - entry->value = NULL; -#ifdef Py_GIL_DISABLED - _PySeqLock_UnlockWrite(&entry->sequence); -#endif - } -} - - -void -_PyType_InitCache(PyInterpreterState *interp) -{ - struct type_cache *cache = &interp->types.type_cache; - for (Py_ssize_t i = 0; i < (1 << MCACHE_SIZE_EXP); i++) { - struct type_cache_entry *entry = &cache->hashtable[i]; - assert(entry->name == NULL); - - entry->version = 0; - // Set to None so _PyType_LookupRef() can use Py_SETREF(), - // rather than using slower Py_XSETREF(). - entry->name = Py_None; - entry->value = NULL; - } -} - - -static unsigned int -_PyType_ClearCache(PyInterpreterState *interp) -{ - struct type_cache *cache = &interp->types.type_cache; - // Set to None, rather than NULL, so _PyType_LookupRef() can - // use Py_SETREF() rather than using slower Py_XSETREF(). - type_cache_clear(cache, Py_None); - - return NEXT_VERSION_TAG(interp) - 1; -} - - unsigned int PyType_ClearCache(void) { PyInterpreterState *interp = _PyInterpreterState_GET(); - return _PyType_ClearCache(interp); + return NEXT_VERSION_TAG(interp) - 1; } void _PyTypes_Fini(PyInterpreterState *interp) { - struct type_cache *cache = &interp->types.type_cache; - type_cache_clear(cache, NULL); - // All the managed static types should have been finalized already. assert(interp->types.for_extensions.num_initialized == 0); for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { @@ -1231,6 +1160,7 @@ _PyType_Modified_Unlocked(PyTypeObject *type) } set_version_unlocked(type, 0); /* 0 is not a valid version tag */ + _PyTypeCache_Invalidate(type); if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -1314,6 +1244,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) clear: assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); set_version_unlocked(type, 0); /* 0 is not a valid version tag */ + _PyTypeCache_Invalidate(type); type->tp_versions_used = _Py_ATTR_CACHE_UNUSED; if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the @@ -6212,69 +6143,6 @@ is_dunder_name(PyObject *name) return 0; } -static PyObject * -update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) -{ - _Py_atomic_store_ptr_relaxed(&entry->value, value); /* borrowed */ - assert(PyUnstable_Unicode_GET_CACHED_HASH(name) != -1); - OBJECT_STAT_INC_COND(type_cache_collisions, entry->name != Py_None && entry->name != name); - // We're releasing this under the lock for simplicity sake because it's always a - // exact unicode object or Py_None so it's safe to do so. - PyObject *old_name = entry->name; - _Py_atomic_store_ptr_relaxed(&entry->name, Py_NewRef(name)); - // We must write the version last to avoid _Py_TryXGetStackRef() - // operating on an invalid (already deallocated) value inside - // _PyType_LookupRefAndVersion(). If we write the version first then a - // reader could pass the "entry_version == type_version" check but could - // be using the old entry value. - _Py_atomic_store_uint32_release(&entry->version, version_tag); - return old_name; -} - -#if Py_GIL_DISABLED - -static void -update_cache_gil_disabled(struct type_cache_entry *entry, PyObject *name, - unsigned int version_tag, PyObject *value) -{ - _PySeqLock_LockWrite(&entry->sequence); - - // update the entry - if (entry->name == name && - entry->value == value && - entry->version == version_tag) { - // We raced with another update, bail and restore previous sequence. - _PySeqLock_AbandonWrite(&entry->sequence); - return; - } - - PyObject *old_value = update_cache(entry, name, version_tag, value); - - // Then update sequence to the next valid value - _PySeqLock_UnlockWrite(&entry->sequence); - - Py_DECREF(old_value); -} - -#endif - -void -_PyTypes_AfterFork(void) -{ -#ifdef Py_GIL_DISABLED - struct type_cache *cache = get_type_cache(); - for (Py_ssize_t i = 0; i < (1 << MCACHE_SIZE_EXP); i++) { - struct type_cache_entry *entry = &cache->hashtable[i]; - if (_PySeqLock_AfterFork(&entry->sequence)) { - // Entry was in the process of updating while forking, clear it... - entry->value = NULL; - Py_SETREF(entry->name, Py_None); - entry->version = 0; - } - } -#endif -} - /* Internal API to look for a name through the MRO. This returns a strong reference, and doesn't set an exception! If nonzero, version is set to the value of type->tp_version at the time of @@ -6305,45 +6173,16 @@ should_assign_version_tag(PyTypeObject *type, PyObject *name, unsigned int versi unsigned int _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out) { - unsigned int h = MCACHE_HASH_METHOD(type, name); - struct type_cache *cache = get_type_cache(); - struct type_cache_entry *entry = &cache->hashtable[h]; -#ifdef Py_GIL_DISABLED - // synchronize-with other writing threads by doing an acquire load on the sequence - while (1) { - uint32_t sequence = _PySeqLock_BeginRead(&entry->sequence); - uint32_t entry_version = _Py_atomic_load_uint32_acquire(&entry->version); - uint32_t type_version = _Py_atomic_load_uint32_acquire(&type->tp_version_tag); - if (entry_version == type_version && - _Py_atomic_load_ptr_relaxed(&entry->name) == name) { + int cacheable = MCACHE_CACHEABLE_NAME(name); + if (cacheable) { + struct _PyTypeCacheLookupResult r = _PyTypeCache_Lookup(type, name); + if (r.cache_hit) { OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); - if (_Py_TryXGetStackRef(&entry->value, out)) { - // If the sequence is still valid then we're done - if (_PySeqLock_EndRead(&entry->sequence, sequence)) { - return entry_version; - } - PyStackRef_XCLOSE(*out); - } - else { - // If we can't incref the object we need to fallback to locking - break; - } - } - else { - // cache miss - break; + *out = r.value; + return r.version_tag; } } -#else - if (entry->version == type->tp_version_tag && entry->name == name) { - assert(type->tp_version_tag); - OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); - OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); - *out = entry->value ? PyStackRef_FromPyObjectNew(entry->value) : PyStackRef_NULL; - return entry->version; - } -#endif OBJECT_STAT_INC_COND(type_cache_misses, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_misses, is_dunder_name(name)); @@ -6354,14 +6193,25 @@ _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef PyInterpreterState *interp = _PyInterpreterState_GET(); unsigned int version_tag = FT_ATOMIC_LOAD_UINT(type->tp_version_tag); - if (should_assign_version_tag(type, name, version_tag)) { + if (cacheable && + (version_tag != 0 || should_assign_version_tag(type, name, version_tag))) + { BEGIN_TYPE_LOCK(); - assign_version_tag(interp, type); version_tag = type->tp_version_tag; + if (version_tag == 0) { + assign_version_tag(interp, type); + version_tag = type->tp_version_tag; + } res = find_name_in_mro(type, name, out); + // find_name_in_mro can release the type lock and another thread can + // modify the type, so we need to check version tag again before caching the result. + if (res >= 0 && version_tag != 0 && version_tag == type->tp_version_tag) { + _PyTypeCache_Insert(type, name, PyStackRef_AsPyObjectBorrow(*out)); + } END_TYPE_LOCK(); } else { + version_tag = 0; res = find_name_in_mro(type, name, out); } @@ -6371,17 +6221,6 @@ _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef return 0; } - if (version_tag == 0 || !MCACHE_CACHEABLE_NAME(name)) { - return 0; - } - - PyObject *res_obj = PyStackRef_AsPyObjectBorrow(*out); -#if Py_GIL_DISABLED - update_cache_gil_disabled(entry, name, version_tag, res_obj); -#else - PyObject *old_value = update_cache(entry, name, version_tag, res_obj); - Py_DECREF(old_value); -#endif return version_tag; } @@ -6840,7 +6679,10 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) done: Py_DECREF(name); Py_XDECREF(descr); - Py_XDECREF(old_value); + // delay decref of the old value as lock-free type cache readers may access it + if (old_value != NULL && !_Py_IsImmortal(old_value)) { + _PyObject_XDecRefDelayed(old_value); + } return res; } @@ -6912,6 +6754,7 @@ clear_static_type_objects(PyInterpreterState *interp, PyTypeObject *type, if (final) { Py_CLEAR(type->tp_cache); } + _PyTypeCache_Invalidate(type); clear_tp_dict(type); clear_tp_bases(type, final); clear_tp_mro(type, final); @@ -7022,6 +6865,7 @@ type_dealloc(PyObject *self) Py_XDECREF(type->tp_mro); Py_XDECREF(type->tp_cache); clear_tp_subclasses(type); + _PyTypeCache_Invalidate(type); /* A type's tp_doc is heap allocated, unlike the tp_doc slots * of most other objects. It's okay to cast it to char *. @@ -9536,6 +9380,8 @@ type_ready(PyTypeObject *type, int initial) goto error; } + _PyTypeCache_InitType(type); + #ifdef Py_TRACE_REFS /* PyType_Ready is the closest thing we have to a choke point * for type objects, so is the best place I can think of to try diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 6562546ae2f9ccb..a4550c0f5f3363a 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -8279,7 +8279,10 @@ _PyUnicode_DecodeIconv(const char *encoding, char *outptr = (char *)chunk; size_t outleft = sizeof(chunk); - size_t ret = iconv(cd, &inptr, &inleft, &outptr, &outleft); + /* Cast the input buffer through void*: iconv() declares its second + argument as "char **" on most systems but "const char **" on some + (e.g. illumos), and void* converts to either without a warning. */ + size_t ret = iconv(cd, (void *)&inptr, &inleft, &outptr, &outleft); int err = errno; in = inptr; @@ -8452,7 +8455,8 @@ _PyUnicode_EncodeIconv(const char *encoding, PyObject *unicode, size_t outleft = (size_t)(outend - out); /* When the whole string is converted, a final iconv() call with a NULL input flushes any pending shift sequence (e.g. ISO-2022). */ - size_t ret = iconv(cd, flushing ? NULL : &inptr, &inleft, &out, &outleft); + /* See the note above on the void* cast of the iconv() input buffer. */ + size_t ret = iconv(cd, flushing ? NULL : (void *)&inptr, &inleft, &out, &outleft); if (!flushing) { up = inptr; } diff --git a/PC/classicAppCompat.can.xml b/PC/classicAppCompat.can.xml deleted file mode 100644 index df361f8e3e2cd32..000000000000000 --- a/PC/classicAppCompat.can.xml +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/PC/classicAppCompat.cat b/PC/classicAppCompat.cat deleted file mode 100644 index 3d213596accffa2..000000000000000 Binary files a/PC/classicAppCompat.cat and /dev/null differ diff --git a/PC/classicAppCompat.sccd b/PC/classicAppCompat.sccd deleted file mode 100644 index d7a70cdd0533c3d..000000000000000 --- a/PC/classicAppCompat.sccd +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - MIIucgYJKoZIhvcNAQcCoIIuYzCCLl8CAQExDzANBglghkgBZQMEAgEFADCCARAGCSsGAQQBgjcKAaCCAQEwgf4wDAYKKwYBBAGCNwwBAQQQ293slyDTMUOmVyeQqmcNSBcNMjIwMjA0MDkwNjU2WjAOBgorBgEEAYI3DAEDBQAwgbwwKgQUdgFpbn9QXi/ly4CZFKA2Eimoq6YxEjAQBgorBgEEAYI3DAIDMQKCADCBjQQg3YHMbvV3unL0mx/RJ8ihTJd1C/SYSnMHbN0yMrWijuQxaTAQBgorBgEEAYI3DAIDMQKCADBVBgorBgEEAYI3AgEEMUcwRTAQBgorBgEEAYI3AgEZogKAADAxMA0GCWCGSAFlAwQCAQUABCDdgcxu9Xe6cvSbH9EnyKFMl3UL9JhKcwds3TIytaKO5KCCE8owggYEMIID7KADAgECAhMzAA/d9uEraWrDsxZGAAAAD932MA0GCSqGSIb3DQEBCwUAMIGLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMQwwCgYDVQQLEwNBT0MxJzAlBgNVBAMTHk1pY3Jvc29mdCBNYXJrZXRwbGFjZSBDQSBHIDAyMjAeFw0yMjAyMDQwODU5MzZaFw0yMjAyMDcwODU5MzZaMC8xLTArBgNVBAMTJDQ2ZTIwYzY2LTUxNTEtNDNhOS05MWQ4LTMwNDY0ZGZhMDg4ZjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALaMR5Ajlp2POidTxT0N47z5sukiE9fqMuI1QimTXIfB2/gPdMsNUbVCD57ha8J1DUJQpO5nuunbk3SONwfDTdWKx9u7zWubpEmgG7hZ8LTKFlF+xhS6lS/FBSSHyXpdScWPg62BMnDysqHMH/AjLw26HNPRWK8A0vx9jsalFwLg15u4MPuKN3Bpawr/OL0B+7eh/dGO3PutAqJ4aZs2lUCIyODg0q3Tzhgi7SIvFacFWJ8Qj6+D3AfasOv8oanfpNLLPhPlxXGEK9sMKHOOb8mTU9V/ibERqEKTHkJ24Vu+BwrXq5eVedtwmHT2WWR7teaIvrUT9AiPmMhx4hIF/0MCAwEAAaOCAbowggG2MBgGA1UdJQEB/wQOMAwGCisGAQQBgjdMHAEwHQYDVR0OBBYEFB4NpkpeSLk/j691tPw97rB4XKDJMA0GA1UdEQQGMASCAlVTMB8GA1UdIwQYMBaAFPeC7EkBPjJR1eFAcP4hAMeQ72IPMF8GA1UdHwRYMFYwVKBSoFCGTmh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY3JsL01pY3Jvc29mdCUyME1hcmtldHBsYWNlJTIwQ0ElMjBHJTIwMDIyLmNybDBsBggrBgEFBQcBAQRgMF4wXAYIKwYBBQUHMAKGUGh0dHA6Ly93d3cubWljcm9zb2Z0LmNvbS9wa2lvcHMvY2VydHMvTWljcm9zb2Z0JTIwTWFya2V0cGxhY2UlMjBDQSUyMEclMjAwMjIuY3J0MAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgXgMDwGCSsGAQQBgjcVBwQvMC0GJSsGAQQBgjcVCIOS9kTqrxCDkY0wgqzgLIKinDE0g+6NOIaE7wACAWQCARcwIAYJKwYBBAGCNxUKAQH/BBAwDjAMBgorBgEEAYI3TBwBMA0GCSqGSIb3DQEBCwUAA4ICAQBAiJuIJeMS+PMrxG+kwPWT7yy7A6P8trchwtXyYep/WXu7xT2tH2ys3wuP4DYG5CY4nKekjTKraQhZGuYj4r4KrUviShe8ZWMIcK4MRduXAU81UOd/fsCq5djQ6NDN4Bzq1TxjwZKiEMEWmXaunj6XAlDSX3SgyyMb/ywmur8VVKO+xFVkOxAqCjNo1VaCv3zEvr3Y6dUwO9gYQui1bAQVQzdxRYUPEJlsFBmbNL1AnZ39r4/9K/6orxavU4qwkj4cXnhtMYKDjTIIbTBrfD4glP6mnmkcZpN0ItfEAVgU9rGpXg84hQLeP83nBs2Y8DrF3bBF9867dJCl90c5rK1DMcelmH3oUAwqZ3U+jIIj3HGyfefAQ6HPL0yiY9OYLXQZUxcZF13m92l5s7dy2C21cIh0W0iTcWPIhNn9cia4Hr7FabTC0RoQkO1bg0PKvPMnoe0AqEVmPKImNDncg538AjPy2qeUxn7+0kcuZWEdKVAPwWfzh5qvSAmYKk4XNBfN2E40GQ0ruRdvMVMFeYBLXKm4SJR6zoiBa5v7qzt1uQqZf6GD4BNFZivBp0P2B9lTubEAU6SmUsazRnEVmekIO+x7gDuXbMS8tv6mb1pbGCtIV/wRcIYNcDkHKTHMbW5xR/OmGTM3mPRxoMQtaWtxIeagZkU2nl+pElA/0h7pIjCCBtcwggS/oAMCAQICCmESRKIAAAAAAAIwDQYJKoZIhvcNAQELBQAwgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xMjAwBgNVBAMTKU1pY3Jvc29mdCBSb290IENlcnRpZmljYXRlIEF1dGhvcml0eSAyMDExMB4XDTExMDMyODIxMDkzOVoXDTMxMDMyODIxMTkzOVowfTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEnMCUGA1UEAxMeTWljcm9zb2Z0IE1hcmtldFBsYWNlIFBDQSAyMDExMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAubUaSwGYVsE3MAnPfvmozUhAB3qxBABgJRW1vDp4+tVinXxD32f7k1K89JQ6zDOgS/iDgULC+yFK1K/1Qjac/0M7P6c8v5LSjnWGlERLa/qY32j46S7SLQcit3g2jgoTTO03eUG+9yHZUTGV/FJdRYB8uXhrznJBa+Y+yGwiQKF+m6XFeBH/KORoKFx+dmMoy9EWJ/m/o9IiUj2kzm9C691+vZ/I2w0Bj93W9SPPkV2PCNHlzgfIAoeajWpHmi38Wi3xZHonkzAVBHxPsCBppOoNsWvmAfUM7eBthkSPvFruekyDCPNEYhfGqgqtqLkoBebXLZCOVybF7wTQaLvse60//3P003icRcCoQYgY4NAqrF7j80o5U7DkeXxcB0xvengsaKgiAaV1DKkRbpe98wCqr1AASvm5rAJUYMU+mXmOieV2EelY2jGrenWe9FQpNXYV1NoWBh0WKoFxttoWYAnF705bIWtSZsz08ZfK6WLX4GXNLcPBlgCzfTm1sdKYASWdBbH2haaNhPapFhQQBJHKwnVW2iXErImhuPi45W3MVTZ5D9ASshZx69cLYY6xAdIa+89Kf/uRrsGOVZfahDuDw+NI183iAyzC8z/QRt2P32LYxP0xrCdqVh+DJo2i4NoE8Uk1usCdbVRuBMBQl/AwpOTq7IMvHGElf65CqzUCAwEAAaOCAUswggFHMBAGCSsGAQQBgjcVAQQDAgEAMB0GA1UdDgQWBBQPU8s/FmEl/mCJHdO5fOiQrbOU0TAZBgkrBgEEAYI3FAIEDB4KAFMAdQBiAEMAQTALBgNVHQ8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAfBgNVHSMEGDAWgBRyLToCMZBDuRQFTuHqp8cx0SOJNDBaBgNVHR8EUzBRME+gTaBLhklodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3JsMF4GCCsGAQUFBwEBBFIwUDBOBggrBgEFBQcwAoZCaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXQyMDExXzIwMTFfMDNfMjIuY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQCjuZmM8ZVNDgp9wHsL4RY8KJ8nLinvxFTphNGCrxaLknkYG5pmMhVlX+UB/tSiW8W13W60nggz9u5xwMx7v/1t/Tgm6g2brVyOKI5A7u6/2SIJwkJKFw953K0YIKVT28w9zl8dSJnmRnyR0G86ncWbF6CLQ6A6lBQ9o2mTGVqDr4m35WKAnc6YxUUM1y74mbzFFZr63VHsCcOp3pXWnUqAY1rb6Q6NX1b3clncKqLFm0EjKHcQ56grTbwuuB7pMdh/IFCJR01MQzQbDtpEisbOeZUi43YVAAHKqI1EO9bRwg3frCjwAbml9MmI4utMW94gWFgvrMxIX+n42RBDIjf3Ot3jkT6gt3XeTTmO9bptgblZimhERdkFRUFpVtkocJeLoGuuzP93uH/Yp032wzRH+XmMgujfZv+vnfllJqxdowoQLx55FxLLeTeYfwi/xMSjZO2gNven3U/3KeSCd1kUOFS3AOrwZ0UNOXJeW5JQC6Vfd1BavFZ6FAta1fMLu3WFvNB+FqeHUaU3ya7rmtxJnzk29DeSqXgGNmVSywBS4NajI5jJIKAA6UhNJlsg8CHYwUOKf5ej8OoQCkbadUxXygAfxCfW2YBbujtI+PoyejRFxWUjYFWO5LeTI62UMyqfOEiqugoYjNxmQZla2s4YHVuqIC34R85FQlg9pKQBsDCCBuMwggTLoAMCAQICEzMAAABLZz0Eed1JFCkAAAAAAEswDQYJKoZIhvcNAQELBQAwfTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEnMCUGA1UEAxMeTWljcm9zb2Z0IE1hcmtldFBsYWNlIFBDQSAyMDExMB4XDTE5MDQxODIwNDUwNFoXDTI0MDQxODIwNDUwNFowgYsxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xDDAKBgNVBAsTA0FPQzEnMCUGA1UEAxMeTWljcm9zb2Z0IE1hcmtldHBsYWNlIENBIEcgMDIyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAzGfV5tDxZ3oPG8aq2OwrBLSRFC61lYvWtkqOdKswlBHVncuoOIzekLIFNsMwluoK0plNZ5w/JPmwkGTGIrt9ftbWQz0k7XRfacXvQQfhCaBgjkybvvDx7CTL0UDEUiXfEEkylUcQKOdm9LApPDP4oO/V8RS4ugkqkjsuVaOyfYv0TytjVVQon72W+wR/AyrBAXgfwzzEb85403GTJRzQMlTw1YqdgS/o9SCsvH9dRZTGlWCatIl+a00eTG5zgeu0xCtqwgERhw9UT6mlPqmp6RqJ8XMgKylh9Ss+jI53EmraAFlelph16kuqF6n3vNnRBfhQ+gmNdtKi9s3jI8Di7ip8hYGPHGGhtcw0AYUg5r/VQhIfXxO45zO7SFvjujX0ji0b3WKB/xD9Sg1KO1fiTpj82ifyPqHvL+iwn5dV98C4ru0dQ0hfK7tA4K5qkW+2gym0fBjGYWq81/smrj6LbPdNqCotgTMU4dCMBhmSJOD9fSiPUybrbHJrHGR5YNs5VJbHH5u1ia8UrgsSrGTb1bh38Tb10matJpBI7NwDWTkV7kMvgNRQsGO/1+dg47PLW3X7TbiZniTPC+oAqOh1CiF+ODnem4Nw1iRw854LN3yqibLHgTi093mCBtPGWZxg2sTcHUX4fo/UHNbSh8bERJznnennm4gEJXAYtpnEFSUCAwEAAaOCAUswggFHMA4GA1UdDwEB/wQEAwIBhjAQBgkrBgEEAYI3FQEEAwIBADAdBgNVHQ4EFgQU94LsSQE+MlHV4UBw/iEAx5DvYg8wGQYJKwYBBAGCNxQCBAweCgBTAHUAYgBDAEEwEgYDVR0TAQH/BAgwBgEB/wIBADAfBgNVHSMEGDAWgBQPU8s/FmEl/mCJHdO5fOiQrbOU0TBXBgNVHR8EUDBOMEygSqBIhkZodHRwOi8vY3JsLm1pY3Jvc29mdC5jb20vcGtpL2NybC9wcm9kdWN0cy9NaWNNYXJQQ0EyMDExXzIwMTEtMDMtMjguY3JsMFsGCCsGAQUFBwEBBE8wTTBLBggrBgEFBQcwAoY/aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNNYXJQQ0EyMDExXzIwMTEtMDMtMjguY3J0MA0GCSqGSIb3DQEBCwUAA4ICAQAbTWps31OnBv77EVpCaU68drubToSKtpw9fYt4MvAhorXGRrRRfS6qou5o45QZAKjwzbOrr33dWxF4P3gCGqZxCotW32p6hlXGP7BnYlVw6242nlX0PahgpyWVWBmi0ElKH8N6HYhJqAW74tUNCzY7ALNMwk/JVEMKpWq+5XqutAEmtP0SdoSZPRwPtdiRjMLS8yyjP4HVAiKJ+m0wV3bLBtCG20BttEGkly7fYa2/E0ps41PLeX3T2yKRgmnGjhD16NUcMDdRpBh7MCXktwpISDEanB8QySJNdErU+NaKukgOW68+Oyk7lGiiZcG3boCGNec9JkM7dQdpA4tneQg0FI/o0mkDZR5aVItbAjRfMgAXpm6Hfu9DqOP+/R0K9v+CwyhwEjF3+SeL8YBAijXHD5YaiWzwZzBDtNQpuOXc91NmF7ifdHVy9IE1EhuIIQsMio3l0Uo87vGdwQ50po8o1F72zi1vfvKfDmszVezk6fF7cy7fZYh8I0ceVbZ4XY4xJ76TdYMItUAoPkvVdDq5B/d4oI7/xHfomRsEw08fuLxCRE8/BcvzSlDpnxu4wL6IULqVwHE+fbQ82Lr5ZDLxa1dYUw8zBGgCnx/ObVPj1RO1CaVi13wCxm8yaHeflKJWw7ZHOLeOGQyMRT+f9cFFk+5fzCA9GcwiSlEyYj4ilzGCGWUwghlhAgEBMIGjMIGLMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMQwwCgYDVQQLEwNBT0MxJzAlBgNVBAMTHk1pY3Jvc29mdCBNYXJrZXRwbGFjZSBDQSBHIDAyMgITMwAP3fbhK2lqw7MWRgAAAA/d9jANBglghkgBZQMEAgEFAKCBjzAYBgkqhkiG9w0BCQMxCwYJKwYBBAGCNwoBMC8GCSqGSIb3DQEJBDEiBCBDgOMhvMousTIzk/P1eC2qGiSJLdSm3brGanFMLa41+DBCBgorBgEEAYI3AgEMMTQwMqAUgBIATQBpAGMAcgBvAHMAbwBmAHShGoAYaHR0cDovL3d3dy5taWNyb3NvZnQuY29tMA0GCSqGSIb3DQEBAQUABIIBAJ8LEwzLh2rDfLOOOodKT10yXoCzHPaQLv9sMwei+650mqV0iHaw9bUolBZAdBnF7JBRjY+yb9AIF7M/Av9pHRKvuLHWDntlo49RrfvNucHaJqPjCMm81h/GAuimPSBMpRtTCW+HHDWrw+HoGbRWQ+N/Vpt//pzj9gAzOzs8W9gU0RMz3ub4Hfgunm/CGACjFu5Hi2sjJbgigNlyRnxjI4+h3J7dGVOfR8hlhRB9YjXDB700F0bzaKqxC9LH7kZbAjiHPgLLCWz7OQ/rTo7wV5/e6v9GCKpamydPy/0MObq4TDi0fn75aVHTJJrzlwt1BJKyVPdAsOhiFUH3Mq4WheKhghcAMIIW/AYKKwYBBAGCNwMDATGCFuwwghboBgkqhkiG9w0BBwKgghbZMIIW1QIBAzEPMA0GCWCGSAFlAwQCAQUAMIIBUQYLKoZIhvcNAQkQAQSgggFABIIBPDCCATgCAQEGCisGAQQBhFkKAwEwMTANBglghkgBZQMEAgEFAAQgr34HNu5gF3j5DAITqq38vc6sNxs6ZqLz9i9zU2oQDtoCBmH66bcTCRgTMjAyMjAyMDQwOTA5NDkuNTkxWjAEgAIB9KCB0KSBzTCByjELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjElMCMGA1UECxMcTWljcm9zb2Z0IEFtZXJpY2EgT3BlcmF0aW9uczEmMCQGA1UECxMdVGhhbGVzIFRTUyBFU046REQ4Qy1FMzM3LTJGQUUxJTAjBgNVBAMTHE1pY3Jvc29mdCBUaW1lLVN0YW1wIFNlcnZpY2WgghFXMIIHDDCCBPSgAwIBAgITMwAAAZwPpk1h0p5LKAABAAABnDANBgkqhkiG9w0BAQsFADB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMDAeFw0yMTEyMDIxOTA1MTlaFw0yMzAyMjgxOTA1MTlaMIHKMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSUwIwYDVQQLExxNaWNyb3NvZnQgQW1lcmljYSBPcGVyYXRpb25zMSYwJAYDVQQLEx1UaGFsZXMgVFNTIEVTTjpERDhDLUUzMzctMkZBRTElMCMGA1UEAxMcTWljcm9zb2Z0IFRpbWUtU3RhbXAgU2VydmljZTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANtSKgwZXUkWP6zrXazTaYq7bco9Q2zvU6MN4ka3GRMX2tJZOK4DxeBiQACL/n7YV/sKTslwpD0f9cPU4rCDX9sfcTWo7XPxdHLQ+WkaGbKKWATsqw69bw8hkJ/bjcp2V2A6vGsvwcqJCh07BK3JPmUtZikyy5PZ8fyTyiKGN7hOWlaIU9oIoucUNoAHQJzLq8h20eNgHUh7eI5k+Kyq4v6810LHuA6EHyKJOZN2xTw5JSkLy0FN5Mhg/OaFrFBl3iag2Tqp4InKLt+Jbh/Jd0etnei2aDHFrmlfPmlRSv5wSNX5zAhgEyRpjmQcz1zp0QaSAefRkMm923/ngU51IbrVbAeHj569SHC9doHgsIxkh0K3lpw582+0ONXcIfIU6nkBT+qADAZ+0dT1uu/gRTBy614QAofjo258TbSX9aOU1SHuAC+3bMoyM7jNdHEJROH+msFDBcmJRl4VKsReI5+S69KUGeLIBhhmnmQ6drF8Ip0ZiO+vhAsD3e9AnqnY7Hcge850I9oKvwuwpVwWnKnwwSGElMz7UvCocmoUMXk7Vn2aNti+bdH28+GQb5EMsqhOmvuZOCRpOWN33G+b3g5unwEP0eTiY+LnWa2AuK43z/pplURJVle29K42QPkOcglB6sjLmNpEpb9basJ72eA0Mlp1LtH3oYZGXsggTfuXAgMBAAGjggE2MIIBMjAdBgNVHQ4EFgQUu2kJZ1Ndjl2112SynL6jGMID+rIwHwYDVR0jBBgwFoAUn6cVXQBeYl2D9OXSZacbUzUZ6XIwXwYDVR0fBFgwVjBUoFKgUIZOaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jcmwvTWljcm9zb2Z0JTIwVGltZS1TdGFtcCUyMFBDQSUyMDIwMTAoMSkuY3JsMGwGCCsGAQUFBwEBBGAwXjBcBggrBgEFBQcwAoZQaHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraW9wcy9jZXJ0cy9NaWNyb3NvZnQlMjBUaW1lLVN0YW1wJTIwUENBJTIwMjAxMCgxKS5jcnQwDAYDVR0TAQH/BAIwADATBgNVHSUEDDAKBggrBgEFBQcDCDANBgkqhkiG9w0BAQsFAAOCAgEApwAqpiMYRzNNYyz3PSbtijbeyCpUXcvIrqA4zPtMIcAk34W9u9mRDndWS+tlR3WwTpr1OgaV1wmc6YFzqK6EGWm903UEsFE7xBJMPXjfdVOPhcJB3vfvA0PX56oobcF2OvNsOSwTB8bi/ns+Cs39Puzs+QSNQZd8iAVBCSvxNCL78dln2RGU1xyB4AKqV9vi4Y/Gfmx2FA+jF0y+YLeob0M40nlSxL0q075t7L6iFRMNr0u8ROhzhDPLl+4ePYfUmyYJoobvydel9anAEsHFlhKl+aXb2ic3yNwbsoPycZJL/vo8OVvYYxCy+/5FrQmAvoW0ZEaBiYcKkzrNWt/hX9r5KgdwL61x0ZiTZopTko6W/58UTefTbhX7Pni0MApH3Pvyt6N0IFap+/LlwFRD1zn7e6ccPTwESnuo/auCmgPznq80OATA7vufsRZPvqeX8jKtsraSNscvNQymEWlcqdXV9hYkjb4T/Qse9cUYaoXg68wFHFuslWfTdPYPLl1vqzlPMnNJpC8KtdioDgcq+y1BaSqSm8EdNfwzT37+/JFtVc3Gs915fDqgPZDgOSzKQIV+fw3aPYt2LET3AbmKKW/r13Oy8cg3+D0D362GQBAJVv0NRI5NowgaCw6oNgWOFPrN72WSEcca/8QQiTGP2XpLiGpRDJZ6sWRpRYNdydkwggdxMIIFWaADAgECAhMzAAAAFcXna54Cm0mZAAAAAAAVMA0GCSqGSIb3DQEBCwUAMIGIMQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMTIwMAYDVQQDEylNaWNyb3NvZnQgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRob3JpdHkgMjAxMDAeFw0yMTA5MzAxODIyMjVaFw0zMDA5MzAxODMyMjVaMHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA5OGmTOe0ciELeaLL1yR5vQ7VgtP97pwHB9KpbE51yMo1V/YBf2xK4OK9uT4XYDP/XE/HZveVU3Fa4n5KWv64NmeFRiMMtY0Tz3cywBAY6GB9alKDRLemjkZrBxTzxXb1hlDcwUTIcVxRMTegCjhuje3XD9gmU3w5YQJ6xKr9cmmvHaus9ja+NSZk2pg7uhp7M62AW36MEBydUv626GIl3GoPz130/o5Tz9bshVZN7928jaTjkY+yOSxRnOlwaQ3KNi1wjjHINSi947SHJMPgyY9+tVSP3PoFVZhtaDuaRr3tpK56KTesy+uDRedGbsoy1cCGMFxPLOJiss254o2I5JasAUq7vnGpF1tnYN74kpEeHT39IM9zfUGaRnXNxF803RKJ1v2lIH1+/NmeRd+2ci/bfV+AutuqfjbsNkz2K26oElHovwUDo9Fzpk03dJQcNIIP8BDyt0cY7afomXw/TNuvXsLz1dhzPUNOwTM5TI4CvEJoLhDqhFFG4tG9ahhaYQFzymeiXtcodgLiMxhy16cg8ML6EgrXY28MyTZki1ugpoMhXV8wdJGUlNi5UPkLiWHzNgY1GIRH29wb0f2y1BzFa/ZcUlFdEtsluq9QBXpsxREdcu+N+VLEhReTwDwV2xo3xwgVGD94q0W29R6HXtqPnhZyacaue7e3PmriLq0CAwEAAaOCAd0wggHZMBIGCSsGAQQBgjcVAQQFAgMBAAEwIwYJKwYBBAGCNxUCBBYEFCqnUv5kxJq+gpE8RjUpzxD/LwTuMB0GA1UdDgQWBBSfpxVdAF5iXYP05dJlpxtTNRnpcjBcBgNVHSAEVTBTMFEGDCsGAQQBgjdMg30BATBBMD8GCCsGAQUFBwIBFjNodHRwOi8vd3d3Lm1pY3Jvc29mdC5jb20vcGtpb3BzL0RvY3MvUmVwb3NpdG9yeS5odG0wEwYDVR0lBAwwCgYIKwYBBQUHAwgwGQYJKwYBBAGCNxQCBAweCgBTAHUAYgBDAEEwCwYDVR0PBAQDAgGGMA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAU1fZWy4/oolxiaNE9lJBb186aGMQwVgYDVR0fBE8wTTBLoEmgR4ZFaHR0cDovL2NybC5taWNyb3NvZnQuY29tL3BraS9jcmwvcHJvZHVjdHMvTWljUm9vQ2VyQXV0XzIwMTAtMDYtMjMuY3JsMFoGCCsGAQUFBwEBBE4wTDBKBggrBgEFBQcwAoY+aHR0cDovL3d3dy5taWNyb3NvZnQuY29tL3BraS9jZXJ0cy9NaWNSb29DZXJBdXRfMjAxMC0wNi0yMy5jcnQwDQYJKoZIhvcNAQELBQADggIBAJ1VffwqreEsH2cBMSRb4Z5yS/ypb+pcFLY+TkdkeLEGk5c9MTO1OdfCcTY/2mRsfNB1OW27DzHkwo/7bNGhlBgi7ulmZzpTTd2YurYeeNg2LpypglYAA7AFvonoaeC6Ce5732pvvinLbtg/SHUB2RjebYIM9W0jVOR4U3UkV7ndn/OOPcbzaN9l9qRWqveVtihVJ9AkvUCgvxm2EhIRXT0n4ECWOKz3+SmJw7wXsFSFQrP8DJ6LGYnn8AtqgcKBGUIZUnWKNsIdw2FzLixre24/LAl4FOmRsqlb30mjdAy87JGA0j3mSj5mO0+7hvoyGtmW9I/2kQH2zsZ0/fZMcm8Qq3UwxTSwethQ/gpY3UA8x1RtnWN0SCyxTkctwRQEcb9k+SS+c23Kjgm9swFXSVRk2XPXfx5bRAGOWhmRaw2fpCjcZxkoJLo4S5pu+yFUa2pFEUep8beuyOiJXk+d0tBMdrVXVAmxaQFEfnyhYWxz/gq77EFmPWn9y8FBSX5+k77L+DvktxW/tM4+pTFRhLy/AsGConsXHRWJjXD+57XQKBqJC4822rpM+Zv/Cuk0+CQ1ZyvgDbjmjJnW4SLq8CdCPSWU5nR0W2rRnj7tfqAxM328y+l7vzhwRNGQ8cirOoo6CGJ/2XBjU02N7oJtpQUQwXEGahC0HVUzWLOhcGbyoYICzjCCAjcCAQEwgfihgdCkgc0wgcoxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJTAjBgNVBAsTHE1pY3Jvc29mdCBBbWVyaWNhIE9wZXJhdGlvbnMxJjAkBgNVBAsTHVRoYWxlcyBUU1MgRVNOOkREOEMtRTMzNy0yRkFFMSUwIwYDVQQDExxNaWNyb3NvZnQgVGltZS1TdGFtcCBTZXJ2aWNloiMKAQEwBwYFKw4DAhoDFQDN2Wnq3fCz9ucStub1zQz7129TQKCBgzCBgKR+MHwxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpXYXNoaW5ndG9uMRAwDgYDVQQHEwdSZWRtb25kMR4wHAYDVQQKExVNaWNyb3NvZnQgQ29ycG9yYXRpb24xJjAkBgNVBAMTHU1pY3Jvc29mdCBUaW1lLVN0YW1wIFBDQSAyMDEwMA0GCSqGSIb3DQEBBQUAAgUA5adiZjAiGA8yMDIyMDIwNDE2MjkyNloYDzIwMjIwMjA1MTYyOTI2WjB3MD0GCisGAQQBhFkKBAExLzAtMAoCBQDlp2JmAgEAMAoCAQACAh92AgH/MAcCAQACAhFfMAoCBQDlqLPmAgEAMDYGCisGAQQBhFkKBAIxKDAmMAwGCisGAQQBhFkKAwKgCjAIAgEAAgMHoSChCjAIAgEAAgMBhqAwDQYJKoZIhvcNAQEFBQADgYEAbKzHReDTwgAKGSkgO/zxCAV/RcxwGpZK4M2bWgwqHVIi+/xiAKdQOSKeMz3UwRO4LI4EN4BjeK+ATpruzOtVGkyA8yzwdJFPv99j4O/Hc5kJSvovfECah62tBB3siNIFaAYNK+BF76k3j4+y9q9lF5yjQ8ZylVYbOfOjlJ2uevsxggQNMIIECQIBATCBkzB8MQswCQYDVQQGEwJVUzETMBEGA1UECBMKV2FzaGluZ3RvbjEQMA4GA1UEBxMHUmVkbW9uZDEeMBwGA1UEChMVTWljcm9zb2Z0IENvcnBvcmF0aW9uMSYwJAYDVQQDEx1NaWNyb3NvZnQgVGltZS1TdGFtcCBQQ0EgMjAxMAITMwAAAZwPpk1h0p5LKAABAAABnDANBglghkgBZQMEAgEFAKCCAUowGgYJKoZIhvcNAQkDMQ0GCyqGSIb3DQEJEAEEMC8GCSqGSIb3DQEJBDEiBCD10XWSucJFd8jLe5PaC9h9/GHXK7ZZ0kb4BGFNqgWk9DCB+gYLKoZIhvcNAQkQAi8xgeowgecwgeQwgb0EIDcPRYUgjSzKOhF39d4QgbRZQgrPO7Lo/qE5GtvSeqa8MIGYMIGApH4wfDELMAkGA1UEBhMCVVMxEzARBgNVBAgTCldhc2hpbmd0b24xEDAOBgNVBAcTB1JlZG1vbmQxHjAcBgNVBAoTFU1pY3Jvc29mdCBDb3Jwb3JhdGlvbjEmMCQGA1UEAxMdTWljcm9zb2Z0IFRpbWUtU3RhbXAgUENBIDIwMTACEzMAAAGcD6ZNYdKeSygAAQAAAZwwIgQgPTEmgx6rn5xpH+xmlpYkcn0dL16GyYRqyG8uRajrfTYwDQYJKoZIhvcNAQELBQAEggIAxmSLisdd5cs+7liXX7vsF8zX3Y1rbtyPZ0M466WBaVULlfeuu9RMzOhWha7bkfYmKs5hXOQNBJrIHxnHbWeDu/V7EVF80IQ7/6qWiVPNcLtMnS8+uyw3Tby2gEhshO/HHOJRqllrgtUGuPl8FoAAPbPKV7X7jrg1kqOWoqpwyiSzt5cN/qijwfTcTIaPeON5wMd8wRT2l5F9e/JDBSq0WuzYb2r0kifv9wq21orm4sQkbg5GgDa/oCr3PQVCNSvPb2G7HANcqJXHeRKyoCeVhX5UaFrCQbdxGFytOc5GzptQKNn18yGg4VYrFpigUcglLbuREqqlPabwZh/QfgKdp4b4uVA6On5HIV5e9XdJ9TXhvDArEQ+Kgyao5xlgb4MkvPO9Thh9UIbw9r3JhQ+uKnCvXyiE2SpTLq1OXTjt/RrLZb+qsZW4nrDRBlBswa4G5OVW/svvI8N2/+YYrSdaCWt4unWSnSxeVUWZkugBmWdeoEJpfs5Qy8NwC+g1TyXcMjolOFTfBB6JYpHCbk6Euh2gKxq5weBb7puxVN4cvawYY2zorp6ZXtAmQGflZTl02nWld+nvDoF/zi6IUmP7PTKvbEVs8T+QkCiIM7cpLMs4NFK29f33lu1iZeGwJAxCYsV/P7geuRfA++bpDutTXjLKMqUTgf00sqJ/5HGuwLE= diff --git a/PC/launcher-usage.txt b/PC/launcher-usage.txt deleted file mode 100644 index aad103509daa283..000000000000000 --- a/PC/launcher-usage.txt +++ /dev/null @@ -1,31 +0,0 @@ -Python Launcher for Windows Version %s - -usage: -%s [launcher-args] [python-args] [script [script-args]] - -Launcher arguments: --2 : Launch the latest Python 2.x version --3 : Launch the latest Python 3.x version --X.Y : Launch the specified Python version - -The above default to an architecture native runtime, but will select any -available. Add a "-32" to the argument to only launch 32-bit runtimes, -or add "-64" to omit 32-bit runtimes (this latter option is deprecated). - -To select a specific runtime, use the -V: options. - --V:TAG : Launch a Python runtime with the specified tag --V:COMPANY/TAG : Launch a Python runtime from the specified company and - with the specified tag - --0 --list : List the available pythons --0p --list-paths : List with paths - -If no options are given but a script is specified, the script is checked for a -shebang line. Otherwise, an active virtual environment or global default will -be selected. - -See https://docs.python.org/using/windows.html#python-launcher-for-windows for -additional configuration. - -The following help text is from Python: diff --git a/PC/launcher.c b/PC/launcher.c deleted file mode 100644 index 5667fc851aa5904..000000000000000 --- a/PC/launcher.c +++ /dev/null @@ -1,2047 +0,0 @@ -/* - * Copyright (C) 2011-2013 Vinay Sajip. - * Licensed to PSF under a contributor agreement. - * - * Based on the work of: - * - * Mark Hammond (original author of Python version) - * Curt Hagenlocher (job management) - */ - -#include -#include -#include -#include - -#define BUFSIZE 256 -#define MSGSIZE 1024 - -/* Build options. */ -#define SKIP_PREFIX -#define SEARCH_PATH - -/* Error codes */ - -#define RC_NO_STD_HANDLES 100 -#define RC_CREATE_PROCESS 101 -#define RC_BAD_VIRTUAL_PATH 102 -#define RC_NO_PYTHON 103 -#define RC_NO_MEMORY 104 -/* - * SCRIPT_WRAPPER is used to choose one of the variants of an executable built - * from this source file. If not defined, the PEP 397 Python launcher is built; - * if defined, a script launcher of the type used by setuptools is built, which - * looks for a script name related to the executable name and runs that script - * with the appropriate Python interpreter. - * - * SCRIPT_WRAPPER should be undefined in the source, and defined in a VS project - * which builds the setuptools-style launcher. - */ -#if defined(SCRIPT_WRAPPER) -#define RC_NO_SCRIPT 105 -#endif -/* - * VENV_REDIRECT is used to choose the variant that looks for an adjacent or - * one-level-higher pyvenv.cfg, and uses its "home" property to locate and - * launch the original python.exe. - */ -#if defined(VENV_REDIRECT) -#define RC_NO_VENV_CFG 106 -#define RC_BAD_VENV_CFG 107 -#endif - -/* Just for now - static definition */ - -static FILE * log_fp = NULL; - -static wchar_t * -skip_whitespace(wchar_t * p) -{ - while (*p && isspace(*p)) - ++p; - return p; -} - -static void -debug(wchar_t * format, ...) -{ - va_list va; - - if (log_fp != NULL) { - va_start(va, format); - vfwprintf_s(log_fp, format, va); - va_end(va); - } -} - -static void -winerror(int rc, wchar_t * message, int size) -{ - FormatMessageW( - FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, rc, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - message, size, NULL); -} - -static void -error(int rc, wchar_t * format, ... ) -{ - va_list va; - wchar_t message[MSGSIZE]; - wchar_t win_message[MSGSIZE]; - int len; - - va_start(va, format); - len = _vsnwprintf_s(message, MSGSIZE, _TRUNCATE, format, va); - va_end(va); - - if (rc == 0) { /* a Windows error */ - winerror(GetLastError(), win_message, MSGSIZE); - if (len >= 0) { - _snwprintf_s(&message[len], MSGSIZE - len, _TRUNCATE, L": %ls", - win_message); - } - } - -#if !defined(_WINDOWS) - fwprintf(stderr, L"%ls\n", message); -#else - MessageBoxW(NULL, message, L"Python Launcher is sorry to say ...", - MB_OK); -#endif - exit(rc); -} - -/* - * This function is here to simplify memory management - * and to treat blank values as if they are absent. - */ -static wchar_t * get_env(wchar_t * key) -{ - /* This is not thread-safe, just like getenv */ - static wchar_t buf[BUFSIZE]; - DWORD result = GetEnvironmentVariableW(key, buf, BUFSIZE); - - if (result >= BUFSIZE) { - /* Large environment variable. Accept some leakage */ - wchar_t *buf2 = (wchar_t*)malloc(sizeof(wchar_t) * (result+1)); - if (buf2 == NULL) { - error(RC_NO_MEMORY, L"Could not allocate environment buffer"); - } - GetEnvironmentVariableW(key, buf2, result); - return buf2; - } - - if (result == 0) - /* Either some error, e.g. ERROR_ENVVAR_NOT_FOUND, - or an empty environment variable. */ - return NULL; - - return buf; -} - -#if defined(Py_DEBUG) -/* Do not define EXECUTABLEPATH_VALUE in debug builds as it'll - never point to the debug build. */ -#if defined(_WINDOWS) - -#define PYTHON_EXECUTABLE L"pythonw_d.exe" - -#else - -#define PYTHON_EXECUTABLE L"python_d.exe" - -#endif -#else -#if defined(_WINDOWS) - -#define PYTHON_EXECUTABLE L"pythonw.exe" -#define EXECUTABLEPATH_VALUE L"WindowedExecutablePath" - -#else - -#define PYTHON_EXECUTABLE L"python.exe" -#define EXECUTABLEPATH_VALUE L"ExecutablePath" - -#endif -#endif - -#define MAX_VERSION_SIZE 8 - -typedef struct { - wchar_t version[MAX_VERSION_SIZE]; /* m.n */ - int bits; /* 32 or 64 */ - wchar_t executable[MAX_PATH]; - wchar_t exe_display[MAX_PATH]; -} INSTALLED_PYTHON; - -/* - * To avoid messing about with heap allocations, just assume we can allocate - * statically and never have to deal with more versions than this. - */ -#define MAX_INSTALLED_PYTHONS 100 - -static INSTALLED_PYTHON installed_pythons[MAX_INSTALLED_PYTHONS]; - -static size_t num_installed_pythons = 0; - -/* - * To hold SOFTWARE\Python\PythonCore\X.Y...\InstallPath - * The version name can be longer than MAX_VERSION_SIZE, but will be - * truncated to just X.Y for comparisons. - */ -#define IP_BASE_SIZE 80 -#define IP_VERSION_SIZE 8 -#define IP_SIZE (IP_BASE_SIZE + IP_VERSION_SIZE) -#define CORE_PATH L"SOFTWARE\\Python\\PythonCore" -/* - * Installations from the Microsoft Store will set the same registry keys, - * but because of a limitation in Windows they cannot be enumerated normally - * (unless you have no other Python installations... which is probably false - * because that's the most likely way to get this launcher!) - * This key is under HKEY_LOCAL_MACHINE - */ -#define LOOKASIDE_PATH L"SOFTWARE\\Microsoft\\AppModel\\Lookaside\\user\\Software\\Python\\PythonCore" - -static wchar_t * location_checks[] = { - L"\\", - L"\\PCbuild\\win32\\", - L"\\PCbuild\\amd64\\", - /* To support early 32bit versions of Python that stuck the build binaries - * directly in PCbuild... */ - L"\\PCbuild\\", - NULL -}; - -static INSTALLED_PYTHON * -find_existing_python(const wchar_t * path) -{ - INSTALLED_PYTHON * result = NULL; - size_t i; - INSTALLED_PYTHON * ip; - - for (i = 0, ip = installed_pythons; i < num_installed_pythons; i++, ip++) { - if (_wcsicmp(path, ip->executable) == 0) { - result = ip; - break; - } - } - return result; -} - -static INSTALLED_PYTHON * -find_existing_python2(int bits, const wchar_t * version) -{ - INSTALLED_PYTHON * result = NULL; - size_t i; - INSTALLED_PYTHON * ip; - - for (i = 0, ip = installed_pythons; i < num_installed_pythons; i++, ip++) { - if (bits == ip->bits && _wcsicmp(version, ip->version) == 0) { - result = ip; - break; - } - } - return result; -} - -static void -_locate_pythons_for_key(HKEY root, LPCWSTR subkey, REGSAM flags, int bits, - int display_name_only) -{ - HKEY core_root, ip_key; - LSTATUS status = RegOpenKeyExW(root, subkey, 0, flags, &core_root); - wchar_t message[MSGSIZE]; - DWORD i; - size_t n; - BOOL ok, append_name; - DWORD type, data_size, attrs; - INSTALLED_PYTHON * ip, * pip; - wchar_t ip_version[IP_VERSION_SIZE]; - wchar_t ip_path[IP_SIZE]; - wchar_t * check; - wchar_t ** checkp; - wchar_t *key_name = (root == HKEY_LOCAL_MACHINE) ? L"HKLM" : L"HKCU"; - - if (status != ERROR_SUCCESS) - debug(L"locate_pythons_for_key: unable to open PythonCore key in %ls\n", - key_name); - else { - ip = &installed_pythons[num_installed_pythons]; - for (i = 0; num_installed_pythons < MAX_INSTALLED_PYTHONS; i++) { - status = RegEnumKeyW(core_root, i, ip_version, IP_VERSION_SIZE); - if (status != ERROR_SUCCESS) { - if (status != ERROR_NO_MORE_ITEMS) { - /* unexpected error */ - winerror(status, message, MSGSIZE); - debug(L"Can't enumerate registry key for version %ls: %ls\n", - ip_version, message); - } - break; - } - else { - wcsncpy_s(ip->version, MAX_VERSION_SIZE, ip_version, - MAX_VERSION_SIZE-1); - /* Still treating version as "x.y" rather than sys.winver - * When PEP 514 tags are properly used, we shouldn't need - * to strip this off here. - */ - check = wcsrchr(ip->version, L'-'); - if (check && !wcscmp(check, L"-32")) { - *check = L'\0'; - } - _snwprintf_s(ip_path, IP_SIZE, _TRUNCATE, - L"%ls\\%ls\\InstallPath", subkey, ip_version); - status = RegOpenKeyExW(root, ip_path, 0, flags, &ip_key); - if (status != ERROR_SUCCESS) { - winerror(status, message, MSGSIZE); - /* Note: 'message' already has a trailing \n*/ - debug(L"%ls\\%ls: %ls", key_name, ip_path, message); - continue; - } - data_size = sizeof(ip->executable) - 1; - append_name = FALSE; -#ifdef EXECUTABLEPATH_VALUE - status = RegQueryValueExW(ip_key, EXECUTABLEPATH_VALUE, NULL, &type, - (LPBYTE)ip->executable, &data_size); -#else - status = ERROR_FILE_NOT_FOUND; /* actual error doesn't matter */ -#endif - if (status != ERROR_SUCCESS || type != REG_SZ || !data_size) { - append_name = TRUE; - data_size = sizeof(ip->executable) - 1; - status = RegQueryValueExW(ip_key, NULL, NULL, &type, - (LPBYTE)ip->executable, &data_size); - if (status != ERROR_SUCCESS) { - winerror(status, message, MSGSIZE); - debug(L"%ls\\%ls: %ls\n", key_name, ip_path, message); - RegCloseKey(ip_key); - continue; - } - } - RegCloseKey(ip_key); - if (type != REG_SZ) { - continue; - } - - data_size = data_size / sizeof(wchar_t) - 1; /* for NUL */ - if (ip->executable[data_size - 1] == L'\\') - --data_size; /* reg value ended in a backslash */ - /* ip->executable is data_size long */ - for (checkp = location_checks; *checkp; ++checkp) { - check = *checkp; - if (append_name) { - _snwprintf_s(&ip->executable[data_size], - MAX_PATH - data_size, - MAX_PATH - data_size, - L"%ls%ls", check, PYTHON_EXECUTABLE); - } - attrs = GetFileAttributesW(ip->executable); - if (attrs == INVALID_FILE_ATTRIBUTES) { - winerror(GetLastError(), message, MSGSIZE); - debug(L"locate_pythons_for_key: %ls: %ls", - ip->executable, message); - } - else if (attrs & FILE_ATTRIBUTE_DIRECTORY) { - debug(L"locate_pythons_for_key: '%ls' is a directory\n", - ip->executable); - } - else if (find_existing_python(ip->executable)) { - debug(L"locate_pythons_for_key: %ls: already found\n", - ip->executable); - } - else { - /* check the executable type. */ - if (bits) { - ip->bits = bits; - } else { - ok = GetBinaryTypeW(ip->executable, &attrs); - if (!ok) { - debug(L"Failure getting binary type: %ls\n", - ip->executable); - } - else { - if (attrs == SCS_64BIT_BINARY) - ip->bits = 64; - else if (attrs == SCS_32BIT_BINARY) - ip->bits = 32; - else - ip->bits = 0; - } - } - if (ip->bits == 0) { - debug(L"locate_pythons_for_key: %ls: \ -invalid binary type: %X\n", - ip->executable, attrs); - } - else { - if (display_name_only) { - /* display just the executable name. This is - * primarily for the Store installs */ - const wchar_t *name = wcsrchr(ip->executable, L'\\'); - if (name) { - wcscpy_s(ip->exe_display, MAX_PATH, name+1); - } - } - if (wcschr(ip->executable, L' ') != NULL) { - /* has spaces, so quote, and set original as - * the display name */ - if (!ip->exe_display[0]) { - wcscpy_s(ip->exe_display, MAX_PATH, ip->executable); - } - n = wcslen(ip->executable); - memmove(&ip->executable[1], - ip->executable, n * sizeof(wchar_t)); - ip->executable[0] = L'\"'; - ip->executable[n + 1] = L'\"'; - ip->executable[n + 2] = L'\0'; - } - debug(L"locate_pythons_for_key: %ls \ -is a %dbit executable\n", - ip->executable, ip->bits); - if (find_existing_python2(ip->bits, ip->version)) { - debug(L"locate_pythons_for_key: %ls-%i: already \ -found\n", ip->version, ip->bits); - } - else { - ++num_installed_pythons; - pip = ip++; - if (num_installed_pythons >= - MAX_INSTALLED_PYTHONS) - break; - } - } - } - } - } - } - RegCloseKey(core_root); - } -} - -static int -compare_pythons(const void * p1, const void * p2) -{ - INSTALLED_PYTHON * ip1 = (INSTALLED_PYTHON *) p1; - INSTALLED_PYTHON * ip2 = (INSTALLED_PYTHON *) p2; - /* note reverse sorting on version */ - int result = CompareStringW(LOCALE_INVARIANT, SORT_DIGITSASNUMBERS, - ip2->version, -1, ip1->version, -1); - switch (result) { - case 0: - error(0, L"CompareStringW failed"); - return 0; - case CSTR_LESS_THAN: - return -1; - case CSTR_EQUAL: - return ip2->bits - ip1->bits; /* 64 before 32 */ - case CSTR_GREATER_THAN: - return 1; - default: - return 0; // This should never be reached. - } -} - -static void -locate_pythons_for_key(HKEY root, REGSAM flags) -{ - _locate_pythons_for_key(root, CORE_PATH, flags, 0, FALSE); -} - -static void -locate_store_pythons(void) -{ -#if defined(_M_X64) - /* 64bit process, so look in native registry */ - _locate_pythons_for_key(HKEY_LOCAL_MACHINE, LOOKASIDE_PATH, - KEY_READ, 64, TRUE); -#else - /* 32bit process, so check that we're on 64bit OS */ - BOOL f64 = FALSE; - if (IsWow64Process(GetCurrentProcess(), &f64) && f64) { - _locate_pythons_for_key(HKEY_LOCAL_MACHINE, LOOKASIDE_PATH, - KEY_READ | KEY_WOW64_64KEY, 64, TRUE); - } -#endif -} - -static void -locate_venv_python(void) -{ - static wchar_t venv_python[MAX_PATH]; - INSTALLED_PYTHON * ip; - wchar_t *virtual_env = get_env(L"VIRTUAL_ENV"); - DWORD attrs; - - /* Check for VIRTUAL_ENV environment variable */ - if (virtual_env == NULL || virtual_env[0] == L'\0') { - return; - } - - /* Check for a python executable in the venv */ - debug(L"Checking for Python executable in virtual env '%ls'\n", virtual_env); - _snwprintf_s(venv_python, MAX_PATH, _TRUNCATE, - L"%ls\\Scripts\\%ls", virtual_env, PYTHON_EXECUTABLE); - attrs = GetFileAttributesW(venv_python); - if (attrs == INVALID_FILE_ATTRIBUTES) { - debug(L"Python executable %ls missing from virtual env\n", venv_python); - return; - } - - ip = &installed_pythons[num_installed_pythons++]; - wcscpy_s(ip->executable, MAX_PATH, venv_python); - ip->bits = 0; - wcscpy_s(ip->version, MAX_VERSION_SIZE, L"venv"); -} - -static void -locate_all_pythons(void) -{ - /* venv Python is highest priority */ - locate_venv_python(); -#if defined(_M_X64) - /* If we are a 64bit process, first hit the 32bit keys. */ - debug(L"locating Pythons in 32bit registry\n"); - locate_pythons_for_key(HKEY_CURRENT_USER, KEY_READ | KEY_WOW64_32KEY); - locate_pythons_for_key(HKEY_LOCAL_MACHINE, KEY_READ | KEY_WOW64_32KEY); -#else - /* If we are a 32bit process on a 64bit Windows, first hit the 64bit keys.*/ - BOOL f64 = FALSE; - if (IsWow64Process(GetCurrentProcess(), &f64) && f64) { - debug(L"locating Pythons in 64bit registry\n"); - locate_pythons_for_key(HKEY_CURRENT_USER, KEY_READ | KEY_WOW64_64KEY); - locate_pythons_for_key(HKEY_LOCAL_MACHINE, KEY_READ | KEY_WOW64_64KEY); - } -#endif - /* now hit the "native" key for this process bittedness. */ - debug(L"locating Pythons in native registry\n"); - locate_pythons_for_key(HKEY_CURRENT_USER, KEY_READ); - locate_pythons_for_key(HKEY_LOCAL_MACHINE, KEY_READ); - /* Store-installed Python is lowest priority */ - locate_store_pythons(); - qsort(installed_pythons, num_installed_pythons, sizeof(INSTALLED_PYTHON), - compare_pythons); -} - -static INSTALLED_PYTHON * -find_python_by_version(wchar_t const * wanted_ver) -{ - INSTALLED_PYTHON * result = NULL; - INSTALLED_PYTHON * ip = installed_pythons; - size_t i, n; - size_t wlen = wcslen(wanted_ver); - int bits = 0; - - if (wcsstr(wanted_ver, L"-32")) { - bits = 32; - wlen -= wcslen(L"-32"); - } - else if (wcsstr(wanted_ver, L"-64")) { /* Added option to select 64 bit explicitly */ - bits = 64; - wlen -= wcslen(L"-64"); - } - for (i = 0; i < num_installed_pythons; i++, ip++) { - n = wcslen(ip->version); - /* - * If wlen is greater than 1, we're probably trying to find a specific - * version and thus want an exact match: 3.1 != 3.10. Otherwise, we - * just want a prefix match. - */ - if ((wlen > 1) && (n != wlen)) { - continue; - } - if (n > wlen) { - n = wlen; - } - if ((wcsncmp(ip->version, wanted_ver, n) == 0) && - /* bits == 0 => don't care */ - ((bits == 0) || (ip->bits == bits))) { - result = ip; - break; - } - } - return result; -} - - -static wchar_t appdata_ini_path[MAX_PATH]; -static wchar_t launcher_ini_path[MAX_PATH]; - -/* - * Get a value either from the environment or a configuration file. - * The key passed in will either be "python", "python2" or "python3". - */ -static wchar_t * -get_configured_value(wchar_t * key) -{ -/* - * Note: this static value is used to return a configured value - * obtained either from the environment or configuration file. - * This should be OK since there wouldn't be any concurrent calls. - */ - static wchar_t configured_value[MSGSIZE]; - wchar_t * result = NULL; - wchar_t * found_in = L"environment"; - DWORD size; - - /* First, search the environment. */ - _snwprintf_s(configured_value, MSGSIZE, _TRUNCATE, L"py_%ls", key); - result = get_env(configured_value); - if (result == NULL && appdata_ini_path[0]) { - /* Not in environment: check local configuration. */ - size = GetPrivateProfileStringW(L"defaults", key, NULL, - configured_value, MSGSIZE, - appdata_ini_path); - if (size > 0) { - result = configured_value; - found_in = appdata_ini_path; - } - } - if (result == NULL && launcher_ini_path[0]) { - /* Not in environment or local: check global configuration. */ - size = GetPrivateProfileStringW(L"defaults", key, NULL, - configured_value, MSGSIZE, - launcher_ini_path); - if (size > 0) { - result = configured_value; - found_in = launcher_ini_path; - } - } - if (result) { - debug(L"found configured value '%ls=%ls' in %ls\n", - key, result, found_in ? found_in : L"(unknown)"); - } else { - debug(L"found no configured value for '%ls'\n", key); - } - return result; -} - -static INSTALLED_PYTHON * -locate_python(wchar_t * wanted_ver, BOOL from_shebang) -{ - static wchar_t config_key [] = { L"pythonX" }; - static wchar_t * last_char = &config_key[sizeof(config_key) / - sizeof(wchar_t) - 2]; - INSTALLED_PYTHON * result = NULL; - size_t n = wcslen(wanted_ver); - wchar_t * configured_value; - - if (num_installed_pythons == 0) - locate_all_pythons(); - - if (n == 1) { /* just major version specified */ - *last_char = *wanted_ver; - configured_value = get_configured_value(config_key); - if (configured_value != NULL) - wanted_ver = configured_value; - } - if (*wanted_ver) { - result = find_python_by_version(wanted_ver); - debug(L"search for Python version '%ls' found ", wanted_ver); - if (result) { - debug(L"'%ls'\n", result->executable); - } else { - debug(L"no interpreter\n"); - } - } - else { - *last_char = L'\0'; /* look for an overall default */ - result = find_python_by_version(L"venv"); - if (result == NULL) { - configured_value = get_configured_value(config_key); - if (configured_value) - result = find_python_by_version(configured_value); - } - /* Not found a value yet - try by major version. - * If we're looking for an interpreter specified in a shebang line, - * we want to try Python 2 first, then Python 3 (for Unix and backward - * compatibility). If we're being called interactively, assume the user - * wants the latest version available, so try Python 3 first, then - * Python 2. - */ - if (result == NULL) - result = find_python_by_version(from_shebang ? L"2" : L"3"); - if (result == NULL) - result = find_python_by_version(from_shebang ? L"3" : L"2"); - debug(L"search for default Python found "); - if (result) { - debug(L"version %ls at '%ls'\n", - result->version, result->executable); - } else { - debug(L"no interpreter\n"); - } - } - return result; -} - -#if defined(SCRIPT_WRAPPER) -/* - * Check for a script located alongside the executable - */ - -#if defined(_WINDOWS) -#define SCRIPT_SUFFIX L"-script.pyw" -#else -#define SCRIPT_SUFFIX L"-script.py" -#endif - -static wchar_t wrapped_script_path[MAX_PATH]; - -/* Locate the script being wrapped. - * - * This code should store the name of the wrapped script in - * wrapped_script_path, or terminate the program with an error if there is no - * valid wrapped script file. - */ -static void -locate_wrapped_script(void) -{ - wchar_t * p; - size_t plen; - DWORD attrs; - - plen = GetModuleFileNameW(NULL, wrapped_script_path, MAX_PATH); - p = wcsrchr(wrapped_script_path, L'.'); - if (p == NULL) { - debug(L"GetModuleFileNameW returned value has no extension: %ls\n", - wrapped_script_path); - error(RC_NO_SCRIPT, L"Wrapper name '%ls' is not valid.", wrapped_script_path); - } - - wcsncpy_s(p, MAX_PATH - (p - wrapped_script_path) + 1, SCRIPT_SUFFIX, _TRUNCATE); - attrs = GetFileAttributesW(wrapped_script_path); - if (attrs == INVALID_FILE_ATTRIBUTES) { - debug(L"File '%ls' non-existent\n", wrapped_script_path); - error(RC_NO_SCRIPT, L"Script file '%ls' is not present.", wrapped_script_path); - } - - debug(L"Using wrapped script file '%ls'\n", wrapped_script_path); -} -#endif - -/* - * Process creation code - */ - -static BOOL -safe_duplicate_handle(HANDLE in, HANDLE * pout) -{ - BOOL ok; - HANDLE process = GetCurrentProcess(); - DWORD rc; - - *pout = NULL; - ok = DuplicateHandle(process, in, process, pout, 0, TRUE, - DUPLICATE_SAME_ACCESS); - if (!ok) { - rc = GetLastError(); - if (rc == ERROR_INVALID_HANDLE) { - debug(L"DuplicateHandle returned ERROR_INVALID_HANDLE\n"); - ok = TRUE; - } - else { - debug(L"DuplicateHandle returned %d\n", rc); - } - } - return ok; -} - -static BOOL WINAPI -ctrl_c_handler(DWORD code) -{ - return TRUE; /* We just ignore all control events. */ -} - -static void -run_child(wchar_t * cmdline) -{ - HANDLE job; - JOBOBJECT_EXTENDED_LIMIT_INFORMATION info; - DWORD rc; - BOOL ok; - STARTUPINFOW si; - PROCESS_INFORMATION pi; - -#if defined(_WINDOWS) - /* - When explorer launches a Windows (GUI) application, it displays - the "app starting" (the "pointer + hourglass") cursor for a number - of seconds, or until the app does something UI-ish (eg, creating a - window, or fetching a message). As this launcher doesn't do this - directly, that cursor remains even after the child process does these - things. We avoid that by doing a simple post+get message. - See http://bugs.python.org/issue17290 - */ - MSG msg; - - PostMessage(0, 0, 0, 0); - GetMessage(&msg, 0, 0, 0); -#endif - - debug(L"run_child: about to run '%ls'\n", cmdline); - job = CreateJobObject(NULL, NULL); - ok = QueryInformationJobObject(job, JobObjectExtendedLimitInformation, - &info, sizeof(info), &rc); - if (!ok || (rc != sizeof(info)) || !job) - error(RC_CREATE_PROCESS, L"Job information querying failed"); - info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | - JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK; - ok = SetInformationJobObject(job, JobObjectExtendedLimitInformation, &info, - sizeof(info)); - if (!ok) - error(RC_CREATE_PROCESS, L"Job information setting failed"); - memset(&si, 0, sizeof(si)); - GetStartupInfoW(&si); - ok = safe_duplicate_handle(GetStdHandle(STD_INPUT_HANDLE), &si.hStdInput); - if (!ok) - error(RC_NO_STD_HANDLES, L"stdin duplication failed"); - ok = safe_duplicate_handle(GetStdHandle(STD_OUTPUT_HANDLE), &si.hStdOutput); - if (!ok) - error(RC_NO_STD_HANDLES, L"stdout duplication failed"); - ok = safe_duplicate_handle(GetStdHandle(STD_ERROR_HANDLE), &si.hStdError); - if (!ok) - error(RC_NO_STD_HANDLES, L"stderr duplication failed"); - - ok = SetConsoleCtrlHandler(ctrl_c_handler, TRUE); - if (!ok) - error(RC_CREATE_PROCESS, L"control handler setting failed"); - - si.dwFlags = STARTF_USESTDHANDLES; - ok = CreateProcessW(NULL, cmdline, NULL, NULL, TRUE, - 0, NULL, NULL, &si, &pi); - if (!ok) - error(RC_CREATE_PROCESS, L"Unable to create process using '%ls'", cmdline); - AssignProcessToJobObject(job, pi.hProcess); - CloseHandle(pi.hThread); - WaitForSingleObjectEx(pi.hProcess, INFINITE, FALSE); - ok = GetExitCodeProcess(pi.hProcess, &rc); - if (!ok) - error(RC_CREATE_PROCESS, L"Failed to get exit code of process"); - debug(L"child process exit code: %d\n", rc); - exit(rc); -} - -static void -invoke_child(wchar_t * executable, wchar_t * suffix, wchar_t * cmdline) -{ - wchar_t * child_command; - size_t child_command_size; - BOOL no_suffix = (suffix == NULL) || (*suffix == L'\0'); - BOOL no_cmdline = (*cmdline == L'\0'); - - if (no_suffix && no_cmdline) - run_child(executable); - else { - if (no_suffix) { - /* add 2 for space separator + terminating NUL. */ - child_command_size = wcslen(executable) + wcslen(cmdline) + 2; - } - else { - /* add 3 for 2 space separators + terminating NUL. */ - child_command_size = wcslen(executable) + wcslen(suffix) + - wcslen(cmdline) + 3; - } - child_command = calloc(child_command_size, sizeof(wchar_t)); - if (child_command == NULL) - error(RC_CREATE_PROCESS, L"unable to allocate %zd bytes for child command.", - child_command_size); - if (no_suffix) - _snwprintf_s(child_command, child_command_size, - child_command_size - 1, L"%ls %ls", - executable, cmdline); - else - _snwprintf_s(child_command, child_command_size, - child_command_size - 1, L"%ls %ls %ls", - executable, suffix, cmdline); - run_child(child_command); - free(child_command); - } -} - -typedef struct { - wchar_t *shebang; - BOOL search; -} SHEBANG; - -static SHEBANG builtin_virtual_paths [] = { - { L"/usr/bin/env python", TRUE }, - { L"/usr/bin/python", FALSE }, - { L"/usr/local/bin/python", FALSE }, - { L"python", FALSE }, - { NULL, FALSE }, -}; - -/* For now, a static array of commands. */ - -#define MAX_COMMANDS 100 - -typedef struct { - wchar_t key[MAX_PATH]; - wchar_t value[MSGSIZE]; -} COMMAND; - -static COMMAND commands[MAX_COMMANDS]; -static int num_commands = 0; - -#if defined(SKIP_PREFIX) - -static wchar_t * builtin_prefixes [] = { - /* These must be in an order that the longest matches should be found, - * i.e. if the prefix is "/usr/bin/env ", it should match that entry - * *before* matching "/usr/bin/". - */ - L"/usr/bin/env ", - L"/usr/bin/", - L"/usr/local/bin/", - NULL -}; - -static wchar_t * skip_prefix(wchar_t * name) -{ - wchar_t ** pp = builtin_prefixes; - wchar_t * result = name; - wchar_t * p; - size_t n; - - for (; p = *pp; pp++) { - n = wcslen(p); - if (_wcsnicmp(p, name, n) == 0) { - result += n; /* skip the prefix */ - if (p[n - 1] == L' ') /* No empty strings in table, so n > 1 */ - result = skip_whitespace(result); - break; - } - } - return result; -} - -#endif - -#if defined(SEARCH_PATH) - -static COMMAND path_command; - -static COMMAND * find_on_path(wchar_t * name) -{ - wchar_t * pathext; - size_t varsize; - wchar_t * context = NULL; - wchar_t * extension; - COMMAND * result = NULL; - DWORD len; - errno_t rc; - - wcscpy_s(path_command.key, MAX_PATH, name); - if (wcschr(name, L'.') != NULL) { - /* assume it has an extension. */ - len = SearchPathW(NULL, name, NULL, MSGSIZE, path_command.value, NULL); - if (len) { - result = &path_command; - } - } - else { - /* No extension - search using registered extensions. */ - rc = _wdupenv_s(&pathext, &varsize, L"PATHEXT"); - if (rc == 0) { - extension = wcstok_s(pathext, L";", &context); - while (extension) { - len = SearchPathW(NULL, name, extension, MSGSIZE, path_command.value, NULL); - if (len) { - result = &path_command; - break; - } - extension = wcstok_s(NULL, L";", &context); - } - free(pathext); - } - } - return result; -} - -#endif - -static COMMAND * find_command(wchar_t * name) -{ - COMMAND * result = NULL; - COMMAND * cp = commands; - int i; - - for (i = 0; i < num_commands; i++, cp++) { - if (_wcsicmp(cp->key, name) == 0) { - result = cp; - break; - } - } -#if defined(SEARCH_PATH) - if (result == NULL) - result = find_on_path(name); -#endif - return result; -} - -static void -update_command(COMMAND * cp, wchar_t * name, wchar_t * cmdline) -{ - wcsncpy_s(cp->key, MAX_PATH, name, _TRUNCATE); - wcsncpy_s(cp->value, MSGSIZE, cmdline, _TRUNCATE); -} - -static void -add_command(wchar_t * name, wchar_t * cmdline) -{ - if (num_commands >= MAX_COMMANDS) { - debug(L"can't add %ls = '%ls': no room\n", name, cmdline); - } - else { - COMMAND * cp = &commands[num_commands++]; - - update_command(cp, name, cmdline); - } -} - -static void -read_config_file(wchar_t * config_path) -{ - wchar_t keynames[MSGSIZE]; - wchar_t value[MSGSIZE]; - DWORD read; - wchar_t * key; - COMMAND * cp; - wchar_t * cmdp; - - read = GetPrivateProfileStringW(L"commands", NULL, NULL, keynames, MSGSIZE, - config_path); - if (read == MSGSIZE - 1) { - debug(L"read_commands: %ls: not enough space for names\n", config_path); - } - key = keynames; - while (*key) { - read = GetPrivateProfileStringW(L"commands", key, NULL, value, MSGSIZE, - config_path); - if (read == MSGSIZE - 1) { - debug(L"read_commands: %ls: not enough space for %ls\n", - config_path, key); - } - cmdp = skip_whitespace(value); - if (*cmdp) { - cp = find_command(key); - if (cp == NULL) - add_command(key, value); - else - update_command(cp, key, value); - } - key += wcslen(key) + 1; - } -} - -static void read_commands(void) -{ - if (launcher_ini_path[0]) - read_config_file(launcher_ini_path); - if (appdata_ini_path[0]) - read_config_file(appdata_ini_path); -} - -static BOOL -parse_shebang(wchar_t * shebang_line, int nchars, wchar_t ** command, - wchar_t ** suffix, BOOL *search) -{ - BOOL rc = FALSE; - SHEBANG * vpp; - size_t plen; - wchar_t * p; - wchar_t zapped; - wchar_t * endp = shebang_line + nchars - 1; - COMMAND * cp; - wchar_t * skipped; - - *command = NULL; /* failure return */ - *suffix = NULL; - *search = FALSE; - - if ((*shebang_line++ == L'#') && (*shebang_line++ == L'!')) { - shebang_line = skip_whitespace(shebang_line); - if (*shebang_line) { - *command = shebang_line; - for (vpp = builtin_virtual_paths; vpp->shebang; ++vpp) { - plen = wcslen(vpp->shebang); - if (wcsncmp(shebang_line, vpp->shebang, plen) == 0) { - rc = TRUE; - *search = vpp->search; - /* We can do this because all builtin commands contain - * "python". - */ - *command = wcsstr(shebang_line, L"python"); - break; - } - } - if (vpp->shebang == NULL) { - /* - * Not found in builtins - look in customized commands. - * - * We can't permanently modify the shebang line in case - * it's not a customized command, but we can temporarily - * stick a NUL after the command while searching for it, - * then put back the char we zapped. - */ -#if defined(SKIP_PREFIX) - skipped = skip_prefix(shebang_line); -#else - skipped = shebang_line; -#endif - p = wcspbrk(skipped, L" \t\r\n"); - if (p != NULL) { - zapped = *p; - *p = L'\0'; - } - cp = find_command(skipped); - if (p != NULL) - *p = zapped; - if (cp != NULL) { - *command = cp->value; - if (p != NULL) - *suffix = skip_whitespace(p); - } - } - /* remove trailing whitespace */ - while ((endp > shebang_line) && isspace(*endp)) - --endp; - if (endp > shebang_line) - endp[1] = L'\0'; - } - } - return rc; -} - -/* #define CP_UTF8 65001 defined in winnls.h */ -#define CP_UTF16LE 1200 -#define CP_UTF16BE 1201 -#define CP_UTF32LE 12000 -#define CP_UTF32BE 12001 - -typedef struct { - int length; - char sequence[4]; - UINT code_page; -} BOM; - -/* - * Strictly, we don't need to handle UTF-16 and UTF-32, since Python itself - * doesn't. Never mind, one day it might - there's no harm leaving it in. - */ -static BOM BOMs[] = { - { 3, { 0xEF, 0xBB, 0xBF }, CP_UTF8 }, /* UTF-8 - keep first */ - /* Test UTF-32LE before UTF-16LE since UTF-16LE BOM is a prefix - * of UTF-32LE BOM. */ - { 4, { 0xFF, 0xFE, 0x00, 0x00 }, CP_UTF32LE }, /* UTF-32LE */ - { 4, { 0x00, 0x00, 0xFE, 0xFF }, CP_UTF32BE }, /* UTF-32BE */ - { 2, { 0xFF, 0xFE }, CP_UTF16LE }, /* UTF-16LE */ - { 2, { 0xFE, 0xFF }, CP_UTF16BE }, /* UTF-16BE */ - { 0 } /* sentinel */ -}; - -static BOM * -find_BOM(char * buffer) -{ -/* - * Look for a BOM in the input and return a pointer to the - * corresponding structure, or NULL if not found. - */ - BOM * result = NULL; - BOM *bom; - - for (bom = BOMs; bom->length; bom++) { - if (strncmp(bom->sequence, buffer, bom->length) == 0) { - result = bom; - break; - } - } - return result; -} - -static char * -find_terminator(char * buffer, int len, BOM *bom) -{ - char * result = NULL; - char * end = buffer + len; - char * p; - char c; - int cp; - - for (p = buffer; p < end; p++) { - c = *p; - if (c == '\r') { - result = p; - break; - } - if (c == '\n') { - result = p; - break; - } - } - if (result != NULL) { - cp = bom->code_page; - - /* adjustments to include all bytes of the char */ - /* no adjustment needed for UTF-8 or big endian */ - if (cp == CP_UTF16LE) - ++result; - else if (cp == CP_UTF32LE) - result += 3; - ++result; /* point just past terminator */ - } - return result; -} - -static BOOL -validate_version(wchar_t * p) -{ - /* - Version information should start with the major version, - Optionally followed by a period and a minor version, - Optionally followed by a minus and one of 32 or 64. - Valid examples: - 2 - 3 - 2.7 - 3.6 - 2.7-32 - The intent is to add to the valid patterns: - 3.10 - 3-32 - 3.6-64 - 3-64 - */ - BOOL result = (p != NULL); /* Default to False if null pointer. */ - - result = result && iswdigit(*p); /* Result = False if first string element is not a digit. */ - - while (result && iswdigit(*p)) /* Require a major version */ - ++p; /* Skip all leading digit(s) */ - if (result && (*p == L'.')) /* Allow . for major minor separator.*/ - { - result = iswdigit(*++p); /* Must be at least one digit */ - while (result && iswdigit(*++p)) ; /* Skip any more Digits */ - } - if (result && (*p == L'-')) { /* Allow - for Bits Separator */ - switch(*++p){ - case L'3': /* 3 is OK */ - result = (*++p == L'2') && !*++p; /* only if followed by 2 and ended.*/ - break; - case L'6': /* 6 is OK */ - result = (*++p == L'4') && !*++p; /* only if followed by 4 and ended.*/ - break; - default: - result = FALSE; - break; - } - } - result = result && !*p; /* Must have reached EOS */ - return result; - -} - -typedef struct { - unsigned short min; - unsigned short max; - wchar_t version[MAX_VERSION_SIZE]; -} PYC_MAGIC; - -static PYC_MAGIC magic_values[] = { - { 50823, 50823, L"2.0" }, - { 60202, 60202, L"2.1" }, - { 60717, 60717, L"2.2" }, - { 62011, 62021, L"2.3" }, - { 62041, 62061, L"2.4" }, - { 62071, 62131, L"2.5" }, - { 62151, 62161, L"2.6" }, - { 62171, 62211, L"2.7" }, - { 3000, 3131, L"3.0" }, - { 3141, 3151, L"3.1" }, - { 3160, 3180, L"3.2" }, - { 3190, 3230, L"3.3" }, - { 3250, 3310, L"3.4" }, - { 3320, 3351, L"3.5" }, - { 3360, 3379, L"3.6" }, - { 3390, 3399, L"3.7" }, - { 3400, 3419, L"3.8" }, - { 3420, 3429, L"3.9" }, - { 3430, 3449, L"3.10" }, - /* Allow 50 magic numbers per version from here on */ - { 3450, 3499, L"3.11" }, - { 3500, 3549, L"3.12" }, - { 3550, 3599, L"3.13" }, - { 3600, 3649, L"3.14" }, - { 3650, 3699, L"3.15" }, - { 3700, 3749, L"3.16" }, - { 0 } -}; - -static INSTALLED_PYTHON * -find_by_magic(unsigned short magic) -{ - INSTALLED_PYTHON * result = NULL; - PYC_MAGIC * mp; - - for (mp = magic_values; mp->min; mp++) { - if ((magic >= mp->min) && (magic <= mp->max)) { - result = locate_python(mp->version, FALSE); - if (result != NULL) - break; - } - } - return result; -} - -static void -maybe_handle_shebang(wchar_t ** argv, wchar_t * cmdline) -{ -/* - * Look for a shebang line in the first argument. If found - * and we spawn a child process, this never returns. If it - * does return then we process the args "normally". - * - * argv[0] might be a filename with a shebang. - */ - FILE * fp; - errno_t rc = _wfopen_s(&fp, *argv, L"rb"); - char buffer[BUFSIZE]; - wchar_t shebang_line[BUFSIZE + 1]; - size_t read; - char *p; - char * start; - char * shebang_alias = (char *) shebang_line; - BOM* bom; - int i, j, nchars = 0; - int header_len; - BOOL is_virt; - BOOL search; - wchar_t * command; - wchar_t * suffix; - COMMAND *cmd = NULL; - INSTALLED_PYTHON * ip; - - if (rc == 0) { - read = fread(buffer, sizeof(char), BUFSIZE, fp); - debug(L"maybe_handle_shebang: read %zd bytes\n", read); - fclose(fp); - - if ((read >= 4) && (buffer[3] == '\n') && (buffer[2] == '\r')) { - ip = find_by_magic((((unsigned char)buffer[1]) << 8 | - (unsigned char)buffer[0]) & 0xFFFF); - if (ip != NULL) { - debug(L"script file is compiled against Python %ls\n", - ip->version); - invoke_child(ip->executable, NULL, cmdline); - } - } - /* Look for BOM */ - bom = find_BOM(buffer); - if (bom == NULL) { - start = buffer; - debug(L"maybe_handle_shebang: BOM not found, using UTF-8\n"); - bom = BOMs; /* points to UTF-8 entry - the default */ - } - else { - debug(L"maybe_handle_shebang: BOM found, code page %u\n", - bom->code_page); - start = &buffer[bom->length]; - } - p = find_terminator(start, BUFSIZE, bom); - /* - * If no CR or LF was found in the heading, - * we assume it's not a shebang file. - */ - if (p == NULL) { - debug(L"maybe_handle_shebang: No line terminator found\n"); - } - else { - /* - * Found line terminator - parse the shebang. - * - * Strictly, we don't need to handle UTF-16 anf UTF-32, - * since Python itself doesn't. - * Never mind, one day it might. - */ - header_len = (int) (p - start); - switch(bom->code_page) { - case CP_UTF8: - nchars = MultiByteToWideChar(bom->code_page, - 0, - start, header_len, shebang_line, - BUFSIZE); - break; - case CP_UTF16BE: - if (header_len % 2 != 0) { - debug(L"maybe_handle_shebang: UTF-16BE, but an odd number \ -of bytes: %d\n", header_len); - /* nchars = 0; Not needed - initialised to 0. */ - } - else { - for (i = header_len; i > 0; i -= 2) { - shebang_alias[i - 1] = start[i - 2]; - shebang_alias[i - 2] = start[i - 1]; - } - nchars = header_len / sizeof(wchar_t); - } - break; - case CP_UTF16LE: - if ((header_len % 2) != 0) { - debug(L"UTF-16LE, but an odd number of bytes: %d\n", - header_len); - /* nchars = 0; Not needed - initialised to 0. */ - } - else { - /* no actual conversion needed. */ - memcpy(shebang_line, start, header_len); - nchars = header_len / sizeof(wchar_t); - } - break; - case CP_UTF32BE: - if (header_len % 4 != 0) { - debug(L"UTF-32BE, but not divisible by 4: %d\n", - header_len); - /* nchars = 0; Not needed - initialised to 0. */ - } - else { - for (i = header_len, j = header_len / 2; i > 0; i -= 4, - j -= 2) { - shebang_alias[j - 1] = start[i - 2]; - shebang_alias[j - 2] = start[i - 1]; - } - nchars = header_len / sizeof(wchar_t); - } - break; - case CP_UTF32LE: - if (header_len % 4 != 0) { - debug(L"UTF-32LE, but not divisible by 4: %d\n", - header_len); - /* nchars = 0; Not needed - initialised to 0. */ - } - else { - for (i = header_len, j = header_len / 2; i > 0; i -= 4, - j -= 2) { - shebang_alias[j - 1] = start[i - 3]; - shebang_alias[j - 2] = start[i - 4]; - } - nchars = header_len / sizeof(wchar_t); - } - break; - } - if (nchars > 0) { - shebang_line[--nchars] = L'\0'; - is_virt = parse_shebang(shebang_line, nchars, &command, - &suffix, &search); - if (command != NULL) { - debug(L"parse_shebang: found command: %ls\n", command); - if (!is_virt) { - invoke_child(command, suffix, cmdline); - } - else { - suffix = wcschr(command, L' '); - if (suffix != NULL) { - *suffix++ = L'\0'; - suffix = skip_whitespace(suffix); - } - if (wcsncmp(command, L"python", 6)) - error(RC_BAD_VIRTUAL_PATH, L"Unknown virtual \ -path '%ls'", command); - command += 6; /* skip past "python" */ - if (search && ((*command == L'\0') || isspace(*command))) { - /* Command is eligible for path search, and there - * is no version specification. - */ - debug(L"searching PATH for python executable\n"); - cmd = find_on_path(PYTHON_EXECUTABLE); - debug(L"Python on path: %ls\n", cmd ? cmd->value : L""); - if (cmd) { - debug(L"located python on PATH: %ls\n", cmd->value); - invoke_child(cmd->value, suffix, cmdline); - /* Exit here, as we have found the command */ - return; - } - /* FALL THROUGH: No python found on PATH, so fall - * back to locating the correct installed python. - */ - } - if (*command && !validate_version(command)) - error(RC_BAD_VIRTUAL_PATH, L"Invalid version \ -specification: '%ls'.\nIn the first line of the script, 'python' needs to be \ -followed by a valid version specifier.\nPlease check the documentation.", - command); - /* TODO could call validate_version(command) */ - ip = locate_python(command, TRUE); - if (ip == NULL) { - error(RC_NO_PYTHON, L"Requested Python version \ -(%ls) is not installed", command); - } - else { - invoke_child(ip->executable, suffix, cmdline); - } - } - } - } - } - } -} - -static wchar_t * -skip_me(wchar_t * cmdline) -{ - BOOL quoted; - wchar_t c; - wchar_t * result = cmdline; - - quoted = cmdline[0] == L'\"'; - if (!quoted) - c = L' '; - else { - c = L'\"'; - ++result; - } - result = wcschr(result, c); - if (result == NULL) /* when, for example, just exe name on command line */ - result = L""; - else { - ++result; /* skip past space or closing quote */ - result = skip_whitespace(result); - } - return result; -} - -static DWORD version_high = 0; -static DWORD version_low = 0; - -static void -get_version_info(wchar_t * version_text, size_t size) -{ - WORD maj, min, rel, bld; - - if (!version_high && !version_low) - wcsncpy_s(version_text, size, L"0.1", _TRUNCATE); /* fallback */ - else { - maj = HIWORD(version_high); - min = LOWORD(version_high); - rel = HIWORD(version_low); - bld = LOWORD(version_low); - _snwprintf_s(version_text, size, _TRUNCATE, L"%d.%d.%d.%d", maj, - min, rel, bld); - } -} - -static void -show_help_text(wchar_t ** argv) -{ - wchar_t version_text [MAX_PATH]; -#if defined(_M_X64) - BOOL canDo64bit = TRUE; -#else - /* If we are a 32bit process on a 64bit Windows, first hit the 64bit keys. */ - BOOL canDo64bit = FALSE; - IsWow64Process(GetCurrentProcess(), &canDo64bit); -#endif - - get_version_info(version_text, MAX_PATH); - fwprintf(stdout, L"\ -Python Launcher for Windows Version %ls\n\n", version_text); - fwprintf(stdout, L"\ -usage:\n\ -%ls [launcher-args] [python-args] [script [script-args]]\n\n", argv[0]); - fputws(L"\ -Launcher arguments:\n\n\ --2 : Launch the latest Python 2.x version\n\ --3 : Launch the latest Python 3.x version\n\ --X.Y : Launch the specified Python version\n", stdout); - if (canDo64bit) { - fputws(L"\ - The above all default to 64 bit if a matching 64 bit python is present.\n\ --X.Y-32: Launch the specified 32bit Python version\n\ --X-32 : Launch the latest 32bit Python X version\n\ --X.Y-64: Launch the specified 64bit Python version\n\ --X-64 : Launch the latest 64bit Python X version", stdout); - } - fputws(L"\n-0 --list : List the available pythons", stdout); - fputws(L"\n-0p --list-paths : List with paths", stdout); - fputws(L"\n\n If no script is specified the specified interpreter is opened.", stdout); - fputws(L"\nIf an exact version is not given, using the latest version can be overridden by", stdout); - fputws(L"\nany of the following, (in priority order):", stdout); - fputws(L"\n An active virtual environment", stdout); - fputws(L"\n A shebang line in the script (if present)", stdout); - fputws(L"\n With -2 or -3 flag a matching PY_PYTHON2 or PY_PYTHON3 Environment variable", stdout); - fputws(L"\n A PY_PYTHON Environment variable", stdout); - fputws(L"\n From [defaults] in py.ini in your %LOCALAPPDATA%\\py.ini", stdout); - fputws(L"\n From [defaults] in py.ini beside py.exe (use `where py` to locate)", stdout); - fputws(L"\n\nThe following help text is from Python:\n\n", stdout); - fflush(stdout); -} - -static BOOL -show_python_list(wchar_t ** argv) -{ - /* - * Display options -0 - */ - INSTALLED_PYTHON * result = NULL; - INSTALLED_PYTHON * ip = installed_pythons; /* List of installed pythons */ - INSTALLED_PYTHON * defpy = locate_python(L"", FALSE); - size_t i = 0; - wchar_t *p = argv[1]; - wchar_t *ver_fmt = L"-%ls-%d"; - wchar_t *fmt = L"\n %ls"; - wchar_t *defind = L" *"; /* Default indicator */ - - /* - * Output informational messages to stderr to keep output - * clean for use in pipes, etc. - */ - fwprintf(stderr, - L"Installed Pythons found by %s Launcher for Windows", argv[0]); - if (!_wcsicmp(p, L"-0p") || !_wcsicmp(p, L"--list-paths")) - fmt = L"\n %-15ls%ls"; /* include path */ - - if (num_installed_pythons == 0) /* We have somehow got here without searching for pythons */ - locate_all_pythons(); /* Find them, Populates installed_pythons */ - - if (num_installed_pythons == 0) /* No pythons found */ - fwprintf(stderr, L"\nNo Installed Pythons Found!"); - else - { - for (i = 0; i < num_installed_pythons; i++, ip++) { - wchar_t version[BUFSIZ]; - if (wcscmp(ip->version, L"venv") == 0) { - wcscpy_s(version, BUFSIZ, L"(venv)"); - } - else { - swprintf_s(version, BUFSIZ, ver_fmt, ip->version, ip->bits); - } - - if (ip->exe_display[0]) { - fwprintf(stdout, fmt, version, ip->exe_display); - } - else { - fwprintf(stdout, fmt, version, ip->executable); - } - /* If there is a default indicate it */ - if (defpy == ip) - fwprintf(stderr, defind); - } - } - - if ((defpy == NULL) && (num_installed_pythons > 0)) - /* We have pythons but none is the default */ - fwprintf(stderr, L"\n\nCan't find a Default Python.\n\n"); - else - fwprintf(stderr, L"\n\n"); /* End with a blank line */ - return FALSE; /* If this has been called we cannot continue */ -} - -#if defined(VENV_REDIRECT) - -static int -find_home_value(const char *buffer, const char **start, DWORD *length) -{ - for (const char *s = strstr(buffer, "home"); s; s = strstr(s + 1, "\nhome")) { - if (*s == '\n') { - ++s; - } - for (int i = 4; i > 0 && *s; --i, ++s); - - while (*s && iswspace(*s)) { - ++s; - } - if (*s != L'=') { - continue; - } - - do { - ++s; - } while (*s && iswspace(*s)); - - *start = s; - char *nl = strchr(s, '\n'); - if (nl) { - *length = (DWORD)((ptrdiff_t)nl - (ptrdiff_t)s); - } else { - *length = (DWORD)strlen(s); - } - return 1; - } - return 0; -} -#endif - -static wchar_t * -wcsdup_pad(const wchar_t *s, int padding, int *newlen) -{ - size_t len = wcslen(s); - len += 1 + padding; - wchar_t *r = (wchar_t *)malloc(len * sizeof(wchar_t)); - if (!r) { - return NULL; - } - if (wcscpy_s(r, len, s)) { - free(r); - return NULL; - } - *newlen = len < MAXINT ? (int)len : MAXINT; - return r; -} - -static wchar_t * -get_process_name(void) -{ - DWORD bufferLen = MAX_PATH; - DWORD len = bufferLen; - wchar_t *r = NULL; - - while (!r) { - r = (wchar_t *)malloc(bufferLen * sizeof(wchar_t)); - if (!r) { - error(RC_NO_MEMORY, L"out of memory"); - return NULL; - } - len = GetModuleFileNameW(NULL, r, bufferLen); - if (len == 0) { - free(r); - error(0, L"Failed to get module name"); - return NULL; - } else if (len == bufferLen && - GetLastError() == ERROR_INSUFFICIENT_BUFFER) { - free(r); - r = NULL; - bufferLen *= 2; - } - } - - return r; -} - -static int -process(int argc, wchar_t ** argv) -{ - wchar_t * wp; - wchar_t * command; - wchar_t * executable; - wchar_t * p; - wchar_t * argv0; - int rc = 0; - INSTALLED_PYTHON * ip; - BOOL valid; - DWORD size, attrs; - wchar_t message[MSGSIZE]; - void * version_data; - VS_FIXEDFILEINFO * file_info; - UINT block_size; -#if defined(VENV_REDIRECT) - wchar_t * venv_cfg_path; - int newlen; -#elif defined(SCRIPT_WRAPPER) - wchar_t * newcommand; - wchar_t * av[2]; - int newlen; - HRESULT hr; - int index; -#else - HRESULT hr; - int index; -#endif - - setvbuf(stderr, (char *)NULL, _IONBF, 0); - wp = get_env(L"PYLAUNCH_DEBUG"); - if ((wp != NULL) && (*wp != L'\0')) - log_fp = stderr; - -#if defined(_M_X64) - debug(L"launcher build: 64bit\n"); -#else - debug(L"launcher build: 32bit\n"); -#endif -#if defined(_WINDOWS) - debug(L"launcher executable: Windows\n"); -#else - debug(L"launcher executable: Console\n"); -#endif -#if !defined(VENV_REDIRECT) - /* Get the local appdata folder (non-roaming) */ - hr = SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, - NULL, 0, appdata_ini_path); - if (hr != S_OK) { - debug(L"SHGetFolderPath failed: %X\n", hr); - appdata_ini_path[0] = L'\0'; - } - else { - wcsncat_s(appdata_ini_path, MAX_PATH, L"\\py.ini", _TRUNCATE); - attrs = GetFileAttributesW(appdata_ini_path); - if (attrs == INVALID_FILE_ATTRIBUTES) { - debug(L"File '%ls' non-existent\n", appdata_ini_path); - appdata_ini_path[0] = L'\0'; - } else { - debug(L"Using local configuration file '%ls'\n", appdata_ini_path); - } - } -#endif - argv0 = get_process_name(); - size = GetFileVersionInfoSizeW(argv0, &size); - if (size == 0) { - winerror(GetLastError(), message, MSGSIZE); - debug(L"GetFileVersionInfoSize failed: %ls\n", message); - } - else { - version_data = malloc(size); - if (version_data) { - valid = GetFileVersionInfoW(argv0, 0, size, - version_data); - if (!valid) - debug(L"GetFileVersionInfo failed: %X\n", GetLastError()); - else { - valid = VerQueryValueW(version_data, L"\\", - (LPVOID *) &file_info, &block_size); - if (!valid) - debug(L"VerQueryValue failed: %X\n", GetLastError()); - else { - version_high = file_info->dwFileVersionMS; - version_low = file_info->dwFileVersionLS; - } - } - free(version_data); - } - } - -#if defined(VENV_REDIRECT) - /* Allocate some extra space for new filenames */ - venv_cfg_path = wcsdup_pad(argv0, 32, &newlen); - if (!venv_cfg_path) { - error(RC_NO_MEMORY, L"Failed to copy module name"); - } - p = wcsrchr(venv_cfg_path, L'\\'); - - if (p == NULL) { - error(RC_NO_VENV_CFG, L"No pyvenv.cfg file"); - } - p[0] = L'\0'; - wcscat_s(venv_cfg_path, newlen, L"\\pyvenv.cfg"); - attrs = GetFileAttributesW(venv_cfg_path); - if (attrs == INVALID_FILE_ATTRIBUTES) { - debug(L"File '%ls' non-existent\n", venv_cfg_path); - p[0] = '\0'; - p = wcsrchr(venv_cfg_path, L'\\'); - if (p != NULL) { - p[0] = '\0'; - wcscat_s(venv_cfg_path, newlen, L"\\pyvenv.cfg"); - attrs = GetFileAttributesW(venv_cfg_path); - if (attrs == INVALID_FILE_ATTRIBUTES) { - debug(L"File '%ls' non-existent\n", venv_cfg_path); - error(RC_NO_VENV_CFG, L"No pyvenv.cfg file"); - } - } - } - debug(L"Using venv configuration file '%ls'\n", venv_cfg_path); -#else - /* Allocate some extra space for new filenames */ - if (wcscpy_s(launcher_ini_path, MAX_PATH, argv0)) { - error(RC_NO_MEMORY, L"Failed to copy module name"); - } - p = wcsrchr(launcher_ini_path, L'\\'); - - if (p == NULL) { - debug(L"GetModuleFileNameW returned value has no backslash: %ls\n", - launcher_ini_path); - launcher_ini_path[0] = L'\0'; - } - else { - p[0] = L'\0'; - wcscat_s(launcher_ini_path, MAX_PATH, L"\\py.ini"); - attrs = GetFileAttributesW(launcher_ini_path); - if (attrs == INVALID_FILE_ATTRIBUTES) { - debug(L"File '%ls' non-existent\n", launcher_ini_path); - launcher_ini_path[0] = L'\0'; - } else { - debug(L"Using global configuration file '%ls'\n", launcher_ini_path); - } - } -#endif - - command = skip_me(GetCommandLineW()); - debug(L"Called with command line: %ls\n", command); - -#if !defined(VENV_REDIRECT) - /* bpo-35811: The __PYVENV_LAUNCHER__ variable is used to - * override sys.executable and locate the original prefix path. - * However, if it is silently inherited by a non-venv Python - * process, that process will believe it is running in the venv - * still. This is the only place where *we* can clear it (that is, - * when py.exe is being used to launch Python), so we do. - */ - SetEnvironmentVariableW(L"__PYVENV_LAUNCHER__", NULL); -#endif - -#if defined(SCRIPT_WRAPPER) - /* The launcher is being used in "script wrapper" mode. - * There should therefore be a Python script named -script.py in - * the same directory as the launcher executable. - * Put the script name into argv as the first (script name) argument. - */ - - /* Get the wrapped script name - if the script is not present, this will - * terminate the program with an error. - */ - locate_wrapped_script(); - - /* Add the wrapped script to the start of command */ - newlen = wcslen(wrapped_script_path) + wcslen(command) + 2; /* ' ' + NUL */ - newcommand = malloc(sizeof(wchar_t) * newlen); - if (!newcommand) { - error(RC_NO_MEMORY, L"Could not allocate new command line"); - } - else { - wcscpy_s(newcommand, newlen, wrapped_script_path); - wcscat_s(newcommand, newlen, L" "); - wcscat_s(newcommand, newlen, command); - debug(L"Running wrapped script with command line '%ls'\n", newcommand); - read_commands(); - av[0] = wrapped_script_path; - av[1] = NULL; - maybe_handle_shebang(av, newcommand); - /* Returns if no shebang line - pass to default processing */ - command = newcommand; - valid = FALSE; - } -#elif defined(VENV_REDIRECT) - { - FILE *f; - char buffer[4096]; /* 4KB should be enough for anybody */ - char *start; - DWORD len, cch, cch_actual; - size_t cb; - if (_wfopen_s(&f, venv_cfg_path, L"r")) { - error(RC_BAD_VENV_CFG, L"Cannot read '%ls'", venv_cfg_path); - } - cb = fread_s(buffer, sizeof(buffer), sizeof(buffer[0]), - sizeof(buffer) / sizeof(buffer[0]), f); - fclose(f); - - if (!find_home_value(buffer, &start, &len)) { - error(RC_BAD_VENV_CFG, L"Cannot find home in '%ls'", - venv_cfg_path); - } - - cch = MultiByteToWideChar(CP_UTF8, 0, start, len, NULL, 0); - if (!cch) { - error(0, L"Cannot determine memory for home path"); - } - cch += (DWORD)wcslen(PYTHON_EXECUTABLE) + 4; /* include sep, null and quotes */ - executable = (wchar_t *)malloc(cch * sizeof(wchar_t)); - if (executable == NULL) { - error(RC_NO_MEMORY, L"A memory allocation failed"); - } - /* start with a quote - we'll skip this ahead, but want it for the final string */ - executable[0] = L'"'; - cch_actual = MultiByteToWideChar(CP_UTF8, 0, start, len, &executable[1], cch - 1); - if (!cch_actual) { - error(RC_BAD_VENV_CFG, L"Cannot decode home path in '%ls'", - venv_cfg_path); - } - cch_actual += 1; /* account for the first quote */ - executable[cch_actual] = L'\0'; - if (executable[cch_actual - 1] != L'\\') { - executable[cch_actual++] = L'\\'; - executable[cch_actual] = L'\0'; - } - if (wcscat_s(&executable[1], cch - 1, PYTHON_EXECUTABLE)) { - error(RC_BAD_VENV_CFG, L"Cannot create executable path from '%ls'", - venv_cfg_path); - } - /* there's no trailing quote, so we only have to skip one character for the test */ - if (GetFileAttributesW(&executable[1]) == INVALID_FILE_ATTRIBUTES) { - error(RC_NO_PYTHON, L"No Python at '%ls'", executable); - } - /* now append the final quote */ - wcscat_s(executable, cch, L"\""); - /* smuggle our original path through */ - if (!SetEnvironmentVariableW(L"__PYVENV_LAUNCHER__", argv0)) { - error(0, L"Failed to set launcher environment"); - } - valid = 1; - } -#else - if (argc <= 1) { - valid = FALSE; - p = NULL; - } - else { - p = argv[1]; - if ((argc == 2) && // list version args - (!wcsncmp(p, L"-0", wcslen(L"-0")) || - !wcsncmp(p, L"--list", wcslen(L"--list")))) - { - show_python_list(argv); - return rc; - } - valid = valid && (*p == L'-') && validate_version(&p[1]); - if (valid) { - ip = locate_python(&p[1], FALSE); - if (ip == NULL) - { - fwprintf(stdout, \ - L"Python %ls not found!\n", &p[1]); - valid = show_python_list(argv); - error(RC_NO_PYTHON, L"Requested Python version (%ls) not \ -installed, use -0 for available pythons", &p[1]); - } - executable = ip->executable; - command += wcslen(p); - command = skip_whitespace(command); - } - else { - for (index = 1; index < argc; ++index) { - if (*argv[index] != L'-') - break; - } - if (index < argc) { - read_commands(); - maybe_handle_shebang(&argv[index], command); - } - } - } -#endif - - if (!valid) { - if ((argc == 2) && (!_wcsicmp(p, L"-h") || !_wcsicmp(p, L"--help"))) - show_help_text(argv); - if ((argc == 2) && - (!_wcsicmp(p, L"-0") || !_wcsicmp(p, L"--list") || - !_wcsicmp(p, L"-0p") || !_wcsicmp(p, L"--list-paths"))) - { - executable = NULL; /* Info call only */ - } - else { - /* look for the default Python */ - ip = locate_python(L"", FALSE); - if (ip == NULL) - error(RC_NO_PYTHON, L"Can't find a default Python."); - executable = ip->executable; - } - } - if (executable != NULL) - invoke_child(executable, NULL, command); - else - rc = RC_NO_PYTHON; - return rc; -} - -#if defined(_WINDOWS) - -int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, - LPWSTR lpstrCmd, int nShow) -{ - return process(__argc, __wargv); -} - -#else - -int cdecl wmain(int argc, wchar_t ** argv) -{ - return process(argc, argv); -} - -#endif diff --git a/PC/launcher2.c b/PC/launcher2.c deleted file mode 100644 index 4dd18c8eb5462e9..000000000000000 --- a/PC/launcher2.c +++ /dev/null @@ -1,2858 +0,0 @@ -/* - * Rewritten Python launcher for Windows - * - * This new rewrite properly handles PEP 514 and allows any registered Python - * runtime to be launched. It also enables auto-install of versions when they - * are requested but no installation can be found. - */ - -#define __STDC_WANT_LIB_EXT1__ 1 - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define MS_WINDOWS -#include "patchlevel.h" - -#define MAXLEN PATHCCH_MAX_CCH -#define MSGSIZE 1024 - -#define RC_NO_STD_HANDLES 100 -#define RC_CREATE_PROCESS 101 -#define RC_BAD_VIRTUAL_PATH 102 -#define RC_NO_PYTHON 103 -#define RC_NO_MEMORY 104 -#define RC_NO_SCRIPT 105 -#define RC_NO_VENV_CFG 106 -#define RC_BAD_VENV_CFG 107 -#define RC_NO_COMMANDLINE 108 -#define RC_INTERNAL_ERROR 109 -#define RC_DUPLICATE_ITEM 110 -#define RC_INSTALLING 111 -#define RC_NO_PYTHON_AT_ALL 112 -#define RC_NO_SHEBANG 113 -#define RC_RECURSIVE_SHEBANG 114 - -static FILE * log_fp = NULL; - -void -debug(wchar_t * format, ...) -{ - va_list va; - - if (log_fp != NULL) { - wchar_t buffer[MAXLEN]; - int r = 0; - va_start(va, format); - r = vswprintf_s(buffer, MAXLEN, format, va); - va_end(va); - - if (r <= 0) { - return; - } - fputws(buffer, log_fp); - while (r && isspace(buffer[r])) { - buffer[r--] = L'\0'; - } - if (buffer[0]) { - OutputDebugStringW(buffer); - } - } -} - - -void -formatWinerror(int rc, wchar_t * message, int size) -{ - FormatMessageW( - FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, rc, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - message, size, NULL); -} - - -void -winerror(int err, wchar_t * format, ... ) -{ - va_list va; - wchar_t message[MSGSIZE]; - wchar_t win_message[MSGSIZE]; - int len; - - if (err == 0) { - err = GetLastError(); - } - - va_start(va, format); - len = _vsnwprintf_s(message, MSGSIZE, _TRUNCATE, format, va); - va_end(va); - - formatWinerror(err, win_message, MSGSIZE); - if (len >= 0) { - _snwprintf_s(&message[len], MSGSIZE - len, _TRUNCATE, L": %s", - win_message); - } - -#if !defined(_WINDOWS) - fwprintf(stderr, L"%s\n", message); -#else - MessageBoxW(NULL, message, L"Python Launcher is sorry to say ...", - MB_OK); -#endif -} - - -void -error(wchar_t * format, ... ) -{ - va_list va; - wchar_t message[MSGSIZE]; - - va_start(va, format); - _vsnwprintf_s(message, MSGSIZE, _TRUNCATE, format, va); - va_end(va); - -#if !defined(_WINDOWS) - fwprintf(stderr, L"%s\n", message); -#else - MessageBoxW(NULL, message, L"Python Launcher is sorry to say ...", - MB_OK); -#endif -} - - -typedef BOOL (*PIsWow64Process2)(HANDLE, USHORT*, USHORT*); - - -USHORT -_getNativeMachine(void) -{ - static USHORT _nativeMachine = IMAGE_FILE_MACHINE_UNKNOWN; - if (_nativeMachine == IMAGE_FILE_MACHINE_UNKNOWN) { - USHORT processMachine; - HMODULE kernel32 = GetModuleHandleW(L"kernel32.dll"); - PIsWow64Process2 IsWow64Process2 = kernel32 ? - (PIsWow64Process2)GetProcAddress(kernel32, "IsWow64Process2") : - NULL; - if (!IsWow64Process2) { - BOOL wow64Process; - if (!IsWow64Process(NULL, &wow64Process)) { - winerror(0, L"Checking process type"); - } else if (wow64Process) { - // We should always be a 32-bit executable, so if running - // under emulation, it must be a 64-bit host. - _nativeMachine = IMAGE_FILE_MACHINE_AMD64; - } else { - // Not running under emulation, and an old enough OS to not - // have IsWow64Process2, so assume it's x86. - _nativeMachine = IMAGE_FILE_MACHINE_I386; - } - } else if (!IsWow64Process2(NULL, &processMachine, &_nativeMachine)) { - winerror(0, L"Checking process type"); - } - } - return _nativeMachine; -} - - -bool -isAMD64Host(void) -{ - return _getNativeMachine() == IMAGE_FILE_MACHINE_AMD64; -} - - -bool -isARM64Host(void) -{ - return _getNativeMachine() == IMAGE_FILE_MACHINE_ARM64; -} - - -bool -isEnvVarSet(const wchar_t *name) -{ - /* only looking for non-empty, which means at least one character - and the null terminator */ - return GetEnvironmentVariableW(name, NULL, 0) >= 2; -} - - -bool -join(wchar_t *buffer, size_t bufferLength, const wchar_t *fragment) -{ - if (SUCCEEDED(PathCchCombineEx(buffer, bufferLength, buffer, fragment, PATHCCH_ALLOW_LONG_PATHS))) { - return true; - } - return false; -} - - -bool -split_parent(wchar_t *buffer, size_t bufferLength) -{ - return SUCCEEDED(PathCchRemoveFileSpec(buffer, bufferLength)); -} - - -int -_compare(const wchar_t *x, int xLen, const wchar_t *y, int yLen) -{ - // Empty strings sort first - if (!x || !xLen) { - return (!y || !yLen) ? 0 : -1; - } else if (!y || !yLen) { - return 1; - } - switch (CompareStringEx( - LOCALE_NAME_INVARIANT, NORM_IGNORECASE | SORT_DIGITSASNUMBERS, - x, xLen, y, yLen, - NULL, NULL, 0 - )) { - case CSTR_LESS_THAN: - return -1; - case CSTR_EQUAL: - return 0; - case CSTR_GREATER_THAN: - return 1; - default: - winerror(0, L"Error comparing '%.*s' and '%.*s' (compare)", xLen, x, yLen, y); - return -1; - } -} - - -int -_compareArgument(const wchar_t *x, int xLen, const wchar_t *y, int yLen) -{ - // Empty strings sort first - if (!x || !xLen) { - return (!y || !yLen) ? 0 : -1; - } else if (!y || !yLen) { - return 1; - } - switch (CompareStringEx( - LOCALE_NAME_INVARIANT, 0, - x, xLen, y, yLen, - NULL, NULL, 0 - )) { - case CSTR_LESS_THAN: - return -1; - case CSTR_EQUAL: - return 0; - case CSTR_GREATER_THAN: - return 1; - default: - winerror(0, L"Error comparing '%.*s' and '%.*s' (compareArgument)", xLen, x, yLen, y); - return -1; - } -} - -int -_comparePath(const wchar_t *x, int xLen, const wchar_t *y, int yLen) -{ - // Empty strings sort first - if (!x || !xLen) { - return !y || !yLen ? 0 : -1; - } else if (!y || !yLen) { - return 1; - } - switch (CompareStringOrdinal(x, xLen, y, yLen, TRUE)) { - case CSTR_LESS_THAN: - return -1; - case CSTR_EQUAL: - return 0; - case CSTR_GREATER_THAN: - return 1; - default: - winerror(0, L"Error comparing '%.*s' and '%.*s' (comparePath)", xLen, x, yLen, y); - return -1; - } -} - - -bool -_startsWith(const wchar_t *x, int xLen, const wchar_t *y, int yLen) -{ - if (!x || !y) { - return false; - } - yLen = yLen < 0 ? (int)wcsnlen_s(y, MAXLEN) : yLen; - xLen = xLen < 0 ? (int)wcsnlen_s(x, MAXLEN) : xLen; - return xLen >= yLen && 0 == _compare(x, yLen, y, yLen); -} - - -bool -_startsWithArgument(const wchar_t *x, int xLen, const wchar_t *y, int yLen) -{ - if (!x || !y) { - return false; - } - yLen = yLen < 0 ? (int)wcsnlen_s(y, MAXLEN) : yLen; - xLen = xLen < 0 ? (int)wcsnlen_s(x, MAXLEN) : xLen; - return xLen >= yLen && 0 == _compareArgument(x, yLen, y, yLen); -} - - -// Unlike regular startsWith, this function requires that the following -// character is either NULL (that is, the entire string matches) or is one of -// the characters in 'separators'. -bool -_startsWithSeparated(const wchar_t *x, int xLen, const wchar_t *y, int yLen, const wchar_t *separators) -{ - if (!x || !y) { - return false; - } - yLen = yLen < 0 ? (int)wcsnlen_s(y, MAXLEN) : yLen; - xLen = xLen < 0 ? (int)wcsnlen_s(x, MAXLEN) : xLen; - if (xLen < yLen) { - return false; - } - if (xLen == yLen) { - return 0 == _compare(x, xLen, y, yLen); - } - return separators && - 0 == _compare(x, yLen, y, yLen) && - wcschr(separators, x[yLen]) != NULL; -} - - - -/******************************************************************************\ - *** HELP TEXT *** -\******************************************************************************/ - - -int -showHelpText(wchar_t ** argv) -{ - // The help text is stored in launcher-usage.txt, which is compiled into - // the launcher and loaded at runtime if needed. - // - // The file must be UTF-8. There are two substitutions: - // %ls - PY_VERSION (as wchar_t*) - // %ls - argv[0] (as wchar_t*) - HRSRC res = FindResourceExW(NULL, L"USAGE", MAKEINTRESOURCE(1), MAKELANGID(LANG_NEUTRAL, SUBLANG_NEUTRAL)); - HGLOBAL resData = res ? LoadResource(NULL, res) : NULL; - const char *usage = resData ? (const char*)LockResource(resData) : NULL; - if (usage == NULL) { - winerror(0, L"Unable to load usage text"); - return RC_INTERNAL_ERROR; - } - - DWORD cbData = SizeofResource(NULL, res); - DWORD cchUsage = MultiByteToWideChar(CP_UTF8, 0, usage, cbData, NULL, 0); - if (!cchUsage) { - winerror(0, L"Unable to preprocess usage text"); - return RC_INTERNAL_ERROR; - } - - cchUsage += 1; - wchar_t *wUsage = (wchar_t*)malloc(cchUsage * sizeof(wchar_t)); - cchUsage = MultiByteToWideChar(CP_UTF8, 0, usage, cbData, wUsage, cchUsage); - if (!cchUsage) { - winerror(0, L"Unable to preprocess usage text"); - free((void *)wUsage); - return RC_INTERNAL_ERROR; - } - // Ensure null termination - wUsage[cchUsage] = L'\0'; - - fwprintf(stdout, wUsage, (L"" PY_VERSION), argv[0]); - fflush(stdout); - - free((void *)wUsage); - - return 0; -} - - -/******************************************************************************\ - *** SEARCH INFO *** -\******************************************************************************/ - - -struct _SearchInfoBuffer { - struct _SearchInfoBuffer *next; - wchar_t buffer[0]; -}; - - -typedef struct { - // the original string, managed by the OS - const wchar_t *originalCmdLine; - // pointer into the cmdline to mark what we've consumed - const wchar_t *restOfCmdLine; - // if known/discovered, the full executable path of our runtime - const wchar_t *executablePath; - // pointer and length into cmdline for the file to check for a - // shebang line, if any. Length can be -1 if the string is null - // terminated. - const wchar_t *scriptFile; - int scriptFileLength; - // pointer and length into cmdline or a static string with the - // name of the target executable. Length can be -1 if the string - // is null terminated. - const wchar_t *executable; - int executableLength; - // pointer and length into a string with additional interpreter - // arguments to include before restOfCmdLine. Length can be -1 if - // the string is null terminated. - const wchar_t *executableArgs; - int executableArgsLength; - // pointer and length into cmdline or a static string with the - // company name for PEP 514 lookup. Length can be -1 if the string - // is null terminated. - const wchar_t *company; - int companyLength; - // pointer and length into cmdline or a static string with the - // tag for PEP 514 lookup. Length can be -1 if the string is - // null terminated. - const wchar_t *tag; - int tagLength; - // if true, treats 'tag' as a non-PEP 514 filter - bool oldStyleTag; - // if true, ignores 'tag' when a high priority environment is found - // gh-92817: This is currently set when a tag is read from configuration, - // the environment, or a shebang, rather than the command line, and the - // only currently possible high priority environment is an active virtual - // environment - bool lowPriorityTag; - // if true, allow PEP 514 lookup to override 'executable' - bool allowExecutableOverride; - // if true, allow a nearby pyvenv.cfg to locate the executable - bool allowPyvenvCfg; - // if true, allow defaults (env/py.ini) to clarify/override tags - bool allowDefaults; - // if true, prefer windowed (console-less) executable - bool windowed; - // if true, only list detected runtimes without launching - bool list; - // if true, only list detected runtimes with paths without launching - bool listPaths; - // if true, display help message before continuing - bool help; - // if set, limits search to registry keys with the specified Company - // This is intended for debugging and testing only - const wchar_t *limitToCompany; - // dynamically allocated buffers to free later - struct _SearchInfoBuffer *_buffer; -} SearchInfo; - - -wchar_t * -allocSearchInfoBuffer(SearchInfo *search, int wcharCount) -{ - struct _SearchInfoBuffer *buffer = (struct _SearchInfoBuffer*)malloc( - sizeof(struct _SearchInfoBuffer) + - wcharCount * sizeof(wchar_t) - ); - if (!buffer) { - return NULL; - } - buffer->next = search->_buffer; - search->_buffer = buffer; - return buffer->buffer; -} - - -void -freeSearchInfo(SearchInfo *search) -{ - struct _SearchInfoBuffer *b = search->_buffer; - search->_buffer = NULL; - while (b) { - struct _SearchInfoBuffer *nextB = b->next; - free((void *)b); - b = nextB; - } -} - - -void -_debugStringAndLength(const wchar_t *s, int len, const wchar_t *name) -{ - if (!s) { - debug(L"%s: (null)\n", name); - } else if (len == 0) { - debug(L"%s: (empty)\n", name); - } else if (len < 0) { - debug(L"%s: %s\n", name, s); - } else { - debug(L"%s: %.*ls\n", name, len, s); - } -} - - -void -dumpSearchInfo(SearchInfo *search) -{ - if (!log_fp) { - return; - } - -#ifdef __clang__ -#define DEBUGNAME(s) L # s -#else -#define DEBUGNAME(s) # s -#endif -#define DEBUG(s) debug(L"SearchInfo." DEBUGNAME(s) L": %s\n", (search->s) ? (search->s) : L"(null)") -#define DEBUG_2(s, sl) _debugStringAndLength((search->s), (search->sl), L"SearchInfo." DEBUGNAME(s)) -#define DEBUG_BOOL(s) debug(L"SearchInfo." DEBUGNAME(s) L": %s\n", (search->s) ? L"True" : L"False") - DEBUG(originalCmdLine); - DEBUG(restOfCmdLine); - DEBUG(executablePath); - DEBUG_2(scriptFile, scriptFileLength); - DEBUG_2(executable, executableLength); - DEBUG_2(executableArgs, executableArgsLength); - DEBUG_2(company, companyLength); - DEBUG_2(tag, tagLength); - DEBUG_BOOL(oldStyleTag); - DEBUG_BOOL(lowPriorityTag); - DEBUG_BOOL(allowDefaults); - DEBUG_BOOL(allowExecutableOverride); - DEBUG_BOOL(windowed); - DEBUG_BOOL(list); - DEBUG_BOOL(listPaths); - DEBUG_BOOL(help); - DEBUG(limitToCompany); -#undef DEBUG_BOOL -#undef DEBUG_2 -#undef DEBUG -#undef DEBUGNAME -} - - -int -findArgv0Length(const wchar_t *buffer, int bufferLength) -{ - // Note: this implements semantics that are only valid for argv0. - // Specifically, there is no escaping of quotes, and quotes within - // the argument have no effect. A quoted argv0 must start and end - // with a double quote character; otherwise, it ends at the first - // ' ' or '\t'. - int quoted = buffer[0] == L'"'; - for (int i = 1; bufferLength < 0 || i < bufferLength; ++i) { - switch (buffer[i]) { - case L'\0': - return i; - case L' ': - case L'\t': - if (!quoted) { - return i; - } - break; - case L'"': - if (quoted) { - return i + 1; - } - break; - } - } - return bufferLength; -} - - -const wchar_t * -findArgv0End(const wchar_t *buffer, int bufferLength) -{ - return &buffer[findArgv0Length(buffer, bufferLength)]; -} - - -/******************************************************************************\ - *** COMMAND-LINE PARSING *** -\******************************************************************************/ - -// Adapted from https://stackoverflow.com/a/65583702 -typedef struct AppExecLinkFile { // For tag IO_REPARSE_TAG_APPEXECLINK - DWORD reparseTag; - WORD reparseDataLength; - WORD reserved; - ULONG version; - wchar_t stringList[MAX_PATH * 4]; // Multistring (Consecutive UTF-16 strings each ending with a NUL) - /* There are normally 4 strings here. Ex: - Package ID: L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe" - Entry Point: L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe!PythonRedirector" - Executable: L"C:\Program Files\WindowsApps\Microsoft.DesktopAppInstaller_1.17.106910_x64__8wekyb3d8bbwe\AppInstallerPythonRedirector.exe" - Applic. Type: L"0" // Integer as ASCII. "0" = Desktop bridge application; Else sandboxed UWP application - */ -} AppExecLinkFile; - - -int -parseCommandLine(SearchInfo *search) -{ - if (!search || !search->originalCmdLine) { - return RC_NO_COMMANDLINE; - } - - const wchar_t *argv0End = findArgv0End(search->originalCmdLine, -1); - const wchar_t *tail = argv0End; // will be start of the executable name - const wchar_t *end = argv0End; // will be end of the executable name - search->restOfCmdLine = argv0End; // will be first space after argv0 - while (--tail != search->originalCmdLine) { - if (*tail == L'"' && end == argv0End) { - // Move the "end" up to the quote, so we also allow moving for - // a period later on. - end = argv0End = tail; - } else if (*tail == L'.' && end == argv0End) { - end = tail; - } else if (*tail == L'\\' || *tail == L'/') { - ++tail; - break; - } - } - if (tail == search->originalCmdLine && tail[0] == L'"') { - ++tail; - } - // Without special cases, we can now fill in the search struct - int tailLen = (int)(end ? (end - tail) : wcsnlen_s(tail, MAXLEN)); - search->executableLength = -1; - - // Our special cases are as follows -#define MATCHES(s) (0 == _comparePath(tail, tailLen, (s), -1)) -#define STARTSWITH(s) _startsWith(tail, tailLen, (s), -1) - if (MATCHES(L"py")) { - search->executable = L"python.exe"; - search->allowExecutableOverride = true; - search->allowDefaults = true; - } else if (MATCHES(L"pyw")) { - search->executable = L"pythonw.exe"; - search->allowExecutableOverride = true; - search->allowDefaults = true; - search->windowed = true; - } else if (MATCHES(L"py_d")) { - search->executable = L"python_d.exe"; - search->allowExecutableOverride = true; - search->allowDefaults = true; - } else if (MATCHES(L"pyw_d")) { - search->executable = L"pythonw_d.exe"; - search->allowExecutableOverride = true; - search->allowDefaults = true; - search->windowed = true; - } else if (STARTSWITH(L"python3")) { - search->executable = L"python.exe"; - search->tag = &tail[6]; - search->tagLength = tailLen - 6; - search->allowExecutableOverride = true; - search->oldStyleTag = true; - search->allowPyvenvCfg = true; - } else if (STARTSWITH(L"pythonw3")) { - search->executable = L"pythonw.exe"; - search->tag = &tail[7]; - search->tagLength = tailLen - 7; - search->allowExecutableOverride = true; - search->oldStyleTag = true; - search->allowPyvenvCfg = true; - search->windowed = true; - } else { - search->executable = tail; - search->executableLength = tailLen; - search->allowPyvenvCfg = true; - } -#undef STARTSWITH -#undef MATCHES - - // First argument might be one of our options. If so, consume it, - // update flags and then set restOfCmdLine. - const wchar_t *arg = search->restOfCmdLine; - while(*arg && isspace(*arg)) { ++arg; } -#define MATCHES(s) (0 == _compareArgument(arg, argLen, (s), -1)) -#define STARTSWITH(s) _startsWithArgument(arg, argLen, (s), -1) - if (*arg && *arg == L'-' && *++arg) { - tail = arg; - while (*tail && !isspace(*tail)) { ++tail; } - int argLen = (int)(tail - arg); - if (argLen > 0) { - if (STARTSWITH(L"2") || STARTSWITH(L"3")) { - // All arguments starting with 2 or 3 are assumed to be version tags - search->tag = arg; - search->tagLength = argLen; - search->oldStyleTag = true; - search->restOfCmdLine = tail; - } else if (STARTSWITH(L"V:") || STARTSWITH(L"-version:")) { - // Arguments starting with 'V:' specify company and/or tag - const wchar_t *argStart = wcschr(arg, L':') + 1; - const wchar_t *tagStart = wcschr(argStart, L'/') ; - if (tagStart) { - search->company = argStart; - search->companyLength = (int)(tagStart - argStart); - search->tag = tagStart + 1; - } else { - search->tag = argStart; - } - search->tagLength = (int)(tail - search->tag); - search->allowDefaults = false; - search->restOfCmdLine = tail; - } else if (MATCHES(L"0") || MATCHES(L"-list")) { - search->list = true; - search->restOfCmdLine = tail; - } else if (MATCHES(L"0p") || MATCHES(L"-list-paths")) { - search->listPaths = true; - search->restOfCmdLine = tail; - } else if (MATCHES(L"h") || MATCHES(L"-help")) { - search->help = true; - // Do not update restOfCmdLine so that we trigger the help - // message from whichever interpreter we select - } - } - } -#undef STARTSWITH -#undef MATCHES - - // Might have a script filename. If it looks like a filename, add - // it to the SearchInfo struct for later reference. - arg = search->restOfCmdLine; - while(*arg && isspace(*arg)) { ++arg; } - if (*arg && *arg != L'-') { - search->scriptFile = arg; - if (*arg == L'"') { - ++search->scriptFile; - while (*++arg && *arg != L'"') { } - } else { - while (*arg && !isspace(*arg)) { ++arg; } - } - search->scriptFileLength = (int)(arg - search->scriptFile); - } - - return 0; -} - - -int -_decodeShebang(SearchInfo *search, const char *buffer, int bufferLength, bool onlyUtf8, wchar_t **decoded, int *decodedLength) -{ - DWORD cp = CP_UTF8; - int wideLen = MultiByteToWideChar(cp, MB_ERR_INVALID_CHARS, buffer, bufferLength, NULL, 0); - if (!wideLen) { - cp = CP_ACP; - wideLen = MultiByteToWideChar(cp, MB_ERR_INVALID_CHARS, buffer, bufferLength, NULL, 0); - if (!wideLen) { - debug(L"# Failed to decode shebang line (0x%08X)\n", GetLastError()); - return RC_BAD_VIRTUAL_PATH; - } - } - wchar_t *b = allocSearchInfoBuffer(search, wideLen + 1); - if (!b) { - return RC_NO_MEMORY; - } - wideLen = MultiByteToWideChar(cp, 0, buffer, bufferLength, b, wideLen + 1); - if (!wideLen) { - debug(L"# Failed to decode shebang line (0x%08X)\n", GetLastError()); - return RC_BAD_VIRTUAL_PATH; - } - b[wideLen] = L'\0'; - *decoded = b; - *decodedLength = wideLen; - return 0; -} - - -bool -_shebangStartsWith(const wchar_t *buffer, int bufferLength, const wchar_t *prefix, const wchar_t **rest, int *firstArgumentLength) -{ - int prefixLength = (int)wcsnlen_s(prefix, MAXLEN); - if (bufferLength < prefixLength || !_startsWithArgument(buffer, bufferLength, prefix, prefixLength)) { - return false; - } - if (rest) { - *rest = &buffer[prefixLength]; - } - if (firstArgumentLength) { - int i = prefixLength; - while (i < bufferLength && !isspace(buffer[i])) { - i += 1; - } - *firstArgumentLength = i - prefixLength; - } - return true; -} - - -int -ensure_no_redirector_stub(wchar_t* filename, wchar_t* buffer) -{ - // Make sure we didn't find a reparse point that will open the Microsoft Store - // If we did, pretend there was no shebang and let normal handling take over - WIN32_FIND_DATAW findData; - HANDLE hFind = FindFirstFileW(buffer, &findData); - if (!hFind) { - // Let normal handling take over - debug(L"# Did not find %s on PATH\n", filename); - return RC_NO_SHEBANG; - } - - FindClose(hFind); - - if (!(findData.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT && - findData.dwReserved0 & IO_REPARSE_TAG_APPEXECLINK)) { - return 0; - } - - HANDLE hReparsePoint = CreateFileW(buffer, 0, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_FLAG_OPEN_REPARSE_POINT, NULL); - if (!hReparsePoint) { - // Let normal handling take over - debug(L"# Did not find %s on PATH\n", filename); - return RC_NO_SHEBANG; - } - - AppExecLinkFile appExecLink; - - if (!DeviceIoControl(hReparsePoint, FSCTL_GET_REPARSE_POINT, NULL, 0, &appExecLink, sizeof(appExecLink), NULL, NULL)) { - // Let normal handling take over - debug(L"# Did not find %s on PATH\n", filename); - CloseHandle(hReparsePoint); - return RC_NO_SHEBANG; - } - - CloseHandle(hReparsePoint); - - const wchar_t* redirectorPackageId = L"Microsoft.DesktopAppInstaller_8wekyb3d8bbwe"; - - if (0 == wcscmp(appExecLink.stringList, redirectorPackageId)) { - debug(L"# ignoring redirector that would launch store\n"); - return RC_NO_SHEBANG; - } - - return 0; -} - - -int -searchPath(SearchInfo *search, const wchar_t *shebang, int shebangLength) -{ - if (isEnvVarSet(L"PYLAUNCHER_NO_SEARCH_PATH")) { - return RC_NO_SHEBANG; - } - - wchar_t *command; - int commandLength; - if (!_shebangStartsWith(shebang, shebangLength, L"/usr/bin/env ", &command, &commandLength)) { - return RC_NO_SHEBANG; - } - - if (!commandLength || commandLength == MAXLEN) { - return RC_BAD_VIRTUAL_PATH; - } - - int lastDot = commandLength; - while (lastDot > 0 && command[lastDot] != L'.') { - lastDot -= 1; - } - if (!lastDot) { - lastDot = commandLength; - } - - wchar_t filename[MAXLEN]; - if (wcsncpy_s(filename, MAXLEN, command, commandLength)) { - return RC_BAD_VIRTUAL_PATH; - } - - const wchar_t *ext = L".exe"; - // If the command already has an extension, we do not want to add it again - if (!lastDot || _comparePath(&filename[lastDot], -1, ext, -1)) { - if (wcscat_s(filename, MAXLEN, L".exe")) { - return RC_BAD_VIRTUAL_PATH; - } - } - - debug(L"# Search PATH for %s\n", filename); - - wchar_t pathVariable[MAXLEN]; - int n = GetEnvironmentVariableW(L"PATH", pathVariable, MAXLEN); - if (!n) { - if (GetLastError() == ERROR_ENVVAR_NOT_FOUND) { - return RC_NO_SHEBANG; - } - winerror(0, L"Failed to read PATH\n", filename); - return RC_INTERNAL_ERROR; - } - - wchar_t buffer[MAXLEN]; - n = SearchPathW(pathVariable, filename, NULL, MAXLEN, buffer, NULL); - if (!n) { - if (GetLastError() == ERROR_FILE_NOT_FOUND) { - debug(L"# Did not find %s on PATH\n", filename); - // If we didn't find it on PATH, let normal handling take over - return RC_NO_SHEBANG; - } - // Other errors should cause us to break - winerror(0, L"Failed to find %s on PATH\n", filename); - return RC_BAD_VIRTUAL_PATH; - } - - int result = ensure_no_redirector_stub(filename, buffer); - if (result) { - return result; - } - - // Check that we aren't going to call ourselves again - // If we are, pretend there was no shebang and let normal handling take over - if (GetModuleFileNameW(NULL, filename, MAXLEN) && - 0 == _comparePath(filename, -1, buffer, -1)) { - debug(L"# ignoring recursive shebang command\n"); - return RC_RECURSIVE_SHEBANG; - } - - wchar_t *buf = allocSearchInfoBuffer(search, n + 1); - if (!buf || wcscpy_s(buf, n + 1, buffer)) { - return RC_NO_MEMORY; - } - - search->executablePath = buf; - search->executableArgs = &command[commandLength]; - search->executableArgsLength = shebangLength - commandLength; - debug(L"# Found %s on PATH\n", buf); - - return 0; -} - - -int -_readIni(const wchar_t *section, const wchar_t *settingName, wchar_t *buffer, int bufferLength) -{ - wchar_t iniPath[MAXLEN]; - int n; - // Check for _PYLAUNCHER_INIDIR override (used for test isolation) - DWORD len = GetEnvironmentVariableW(L"_PYLAUNCHER_INIDIR", iniPath, MAXLEN); - if (len && len < MAXLEN) { - if (join(iniPath, MAXLEN, L"py.ini")) { - debug(L"# Reading from %s for %s/%s\n", iniPath, section, settingName); - n = GetPrivateProfileStringW(section, settingName, NULL, buffer, bufferLength, iniPath); - if (n) { - debug(L"# Found %s in %s\n", settingName, iniPath); - return n; - } - } - // When _PYLAUNCHER_INIDIR is set, skip the default locations - return 0; - } - if (SUCCEEDED(SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, iniPath)) && - join(iniPath, MAXLEN, L"py.ini")) { - debug(L"# Reading from %s for %s/%s\n", iniPath, section, settingName); - n = GetPrivateProfileStringW(section, settingName, NULL, buffer, bufferLength, iniPath); - if (n) { - debug(L"# Found %s in %s\n", settingName, iniPath); - return n; - } else if (GetLastError() == ERROR_FILE_NOT_FOUND) { - debug(L"# Did not find file %s\n", iniPath); - } else { - winerror(0, L"Failed to read from %s\n", iniPath); - } - } - if (GetModuleFileNameW(NULL, iniPath, MAXLEN) && - SUCCEEDED(PathCchRemoveFileSpec(iniPath, MAXLEN)) && - join(iniPath, MAXLEN, L"py.ini")) { - debug(L"# Reading from %s for %s/%s\n", iniPath, section, settingName); - n = GetPrivateProfileStringW(section, settingName, NULL, buffer, MAXLEN, iniPath); - if (n) { - debug(L"# Found %s in %s\n", settingName, iniPath); - return n; - } else if (GetLastError() == ERROR_FILE_NOT_FOUND) { - debug(L"# Did not find file %s\n", iniPath); - } else { - winerror(0, L"Failed to read from %s\n", iniPath); - } - } - return 0; -} - - -bool -_findCommand(SearchInfo *search, const wchar_t *command, int commandLength) -{ - wchar_t commandBuffer[MAXLEN]; - wchar_t buffer[MAXLEN]; - wcsncpy_s(commandBuffer, MAXLEN, command, commandLength); - int n = _readIni(L"commands", commandBuffer, buffer, MAXLEN); - if (!n) { - return false; - } - wchar_t *path = allocSearchInfoBuffer(search, n + 1); - if (!path) { - return false; - } - wcscpy_s(path, n + 1, buffer); - search->executablePath = path; - return true; -} - - -int -_useShebangAsExecutable(SearchInfo *search, const wchar_t *shebang, int shebangLength) -{ - wchar_t buffer[MAXLEN]; - wchar_t script[MAXLEN]; - wchar_t command[MAXLEN]; - - int commandLength = 0; - int inQuote = 0; - - if (!shebang || !shebangLength) { - return 0; - } - - wchar_t *pC = command; - for (int i = 0; i < shebangLength; ++i) { - wchar_t c = shebang[i]; - if (isspace(c) && !inQuote) { - commandLength = i; - break; - } else if (c == L'"') { - inQuote = !inQuote; - } else if (c == L'/' || c == L'\\') { - *pC++ = L'\\'; - } else { - *pC++ = c; - } - } - *pC = L'\0'; - - if (!GetCurrentDirectoryW(MAXLEN, buffer) || - wcsncpy_s(script, MAXLEN, search->scriptFile, search->scriptFileLength) || - FAILED(PathCchCombineEx(buffer, MAXLEN, buffer, script, - PATHCCH_ALLOW_LONG_PATHS)) || - FAILED(PathCchRemoveFileSpec(buffer, MAXLEN)) || - FAILED(PathCchCombineEx(buffer, MAXLEN, buffer, command, - PATHCCH_ALLOW_LONG_PATHS)) - ) { - return RC_NO_MEMORY; - } - - int n = (int)wcsnlen(buffer, MAXLEN); - wchar_t *path = allocSearchInfoBuffer(search, n + 1); - if (!path) { - return RC_NO_MEMORY; - } - wcscpy_s(path, n + 1, buffer); - search->executablePath = path; - if (commandLength) { - search->executableArgs = &shebang[commandLength]; - search->executableArgsLength = shebangLength - commandLength; - } - return 0; -} - - -int -checkShebang(SearchInfo *search) -{ - // Do not check shebang if a tag was provided or if no script file - // was found on the command line. - if (search->tag || !search->scriptFile) { - return 0; - } - - if (search->scriptFileLength < 0) { - search->scriptFileLength = (int)wcsnlen_s(search->scriptFile, MAXLEN); - } - - wchar_t *scriptFile = (wchar_t*)malloc(sizeof(wchar_t) * (search->scriptFileLength + 1)); - if (!scriptFile) { - return RC_NO_MEMORY; - } - - wcsncpy_s(scriptFile, search->scriptFileLength + 1, - search->scriptFile, search->scriptFileLength); - - HANDLE hFile = CreateFileW(scriptFile, GENERIC_READ, - FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, - NULL, OPEN_EXISTING, 0, NULL); - - if (hFile == INVALID_HANDLE_VALUE) { - debug(L"# Failed to open %s for shebang parsing (0x%08X)\n", - scriptFile, GetLastError()); - free(scriptFile); - return RC_NO_SCRIPT; - } - - DWORD bytesRead = 0; - unsigned char buffer[4096]; - if (!ReadFile(hFile, buffer, sizeof(buffer), &bytesRead, NULL)) { - debug(L"# Failed to read %s for shebang parsing (0x%08X)\n", - scriptFile, GetLastError()); - free(scriptFile); - return 0; - } - - CloseHandle(hFile); - debug(L"# Read %d bytes from %s to find shebang line\n", bytesRead, scriptFile); - free(scriptFile); - - - unsigned char *b = buffer; - bool onlyUtf8 = false; - if (bytesRead > 3 && *b == 0xEF) { - if (*++b == 0xBB && *++b == 0xBF) { - // Allow a UTF-8 BOM - ++b; - bytesRead -= 3; - onlyUtf8 = true; - } else { - debug(L"# Invalid BOM in shebang line"); - return 0; - } - } - if (bytesRead <= 2 || b[0] != '#' || b[1] != '!') { - // No shebang (#!) at start of line - debug(L"# No valid shebang line"); - return 0; - } - ++b; - --bytesRead; - while (--bytesRead > 0 && isspace(*++b)) { } - const unsigned char *start = b; - while (--bytesRead > 0 && *++b != '\r' && *b != '\n') { } - wchar_t *shebang; - int shebangLength; - // We add 1 when bytesRead==0, as in that case we hit EOF and b points - // to the last character in the file, not the newline - int exitCode = _decodeShebang(search, (const char*)start, (int)(b - start + (bytesRead == 0)), onlyUtf8, &shebang, &shebangLength); - if (exitCode) { - return exitCode; - } - debug(L"Shebang: %s\n", shebang); - - // Handle shebangs that we should search PATH for - int executablePathWasSetByUsrBinEnv = 0; - exitCode = searchPath(search, shebang, shebangLength); - if (exitCode == 0) { - executablePathWasSetByUsrBinEnv = 1; - } else if (exitCode != RC_NO_SHEBANG) { - return exitCode; - } - - // Handle some known, case-sensitive shebangs - const wchar_t *command; - int commandLength; - // Each template must end with "python" - static const wchar_t *shebangTemplates[] = { - L"/usr/bin/env python", - L"/usr/bin/python", - L"/usr/local/bin/python", - L"python", - NULL - }; - - for (const wchar_t **tmpl = shebangTemplates; *tmpl; ++tmpl) { - // Just to make sure we don't mess this up in the future - assert(0 == wcscmp(L"python", (*tmpl) + wcslen(*tmpl) - 6)); - - if (_shebangStartsWith(shebang, shebangLength, *tmpl, &command, &commandLength)) { - // Search for "python{command}" overrides. All templates end with - // "python", so we prepend it by jumping back 6 characters - if (_findCommand(search, &command[-6], commandLength + 6)) { - search->executableArgs = &command[commandLength]; - search->executableArgsLength = shebangLength - commandLength; - debug(L"# Treating shebang command '%.*s' as %s\n", - commandLength + 6, &command[-6], search->executablePath); - return 0; - } - - search->tag = command; - search->tagLength = commandLength; - // If we had 'python3.12.exe' then we want to strip the suffix - // off of the tag - if (search->tagLength >= 4) { - const wchar_t *suffix = &search->tag[search->tagLength - 4]; - if (0 == _comparePath(suffix, 4, L".exe", -1)) { - search->tagLength -= 4; - } - } - // If we had 'python3_d' then we want to strip the '_d' (any - // '.exe' is already gone) - if (search->tagLength >= 2) { - const wchar_t *suffix = &search->tag[search->tagLength - 2]; - if (0 == _comparePath(suffix, 2, L"_d", -1)) { - search->tagLength -= 2; - } - } - search->oldStyleTag = true; - search->lowPriorityTag = true; - search->executableArgs = &command[commandLength]; - search->executableArgsLength = shebangLength - commandLength; - if (search->tag && search->tagLength) { - debug(L"# Treating shebang command '%.*s' as 'py -%.*s'\n", - commandLength, command, search->tagLength, search->tag); - } else { - debug(L"# Treating shebang command '%.*s' as 'py'\n", - commandLength, command); - } - return 0; - } - } - - // Didn't match a template, but we found it on PATH - if (executablePathWasSetByUsrBinEnv) { - return 0; - } - - // Unrecognised executables are first tried as command aliases - commandLength = 0; - while (commandLength < shebangLength && !isspace(shebang[commandLength])) { - commandLength += 1; - } - if (_findCommand(search, shebang, commandLength)) { - search->executableArgs = &shebang[commandLength]; - search->executableArgsLength = shebangLength - commandLength; - debug(L"# Treating shebang command '%.*s' as %s\n", - commandLength, shebang, search->executablePath); - return 0; - } - - // Unrecognised commands are joined to the script's directory and treated - // as the executable path - return _useShebangAsExecutable(search, shebang, shebangLength); -} - - -int -checkDefaults(SearchInfo *search) -{ - if (!search->allowDefaults) { - return 0; - } - - // Only resolve old-style (or absent) tags to defaults - if (search->tag && search->tagLength && !search->oldStyleTag) { - return 0; - } - - // If tag is only a major version number, expand it from the environment - // or an ini file - const wchar_t *iniSettingName = NULL; - const wchar_t *envSettingName = NULL; - if (!search->tag || !search->tagLength) { - iniSettingName = L"python"; - envSettingName = L"py_python"; - } else if (0 == wcsncmp(search->tag, L"3", search->tagLength)) { - iniSettingName = L"python3"; - envSettingName = L"py_python3"; - } else if (0 == wcsncmp(search->tag, L"2", search->tagLength)) { - iniSettingName = L"python2"; - envSettingName = L"py_python2"; - } else { - debug(L"# Cannot select defaults for tag '%.*s'\n", search->tagLength, search->tag); - return 0; - } - - // First, try to read an environment variable - wchar_t buffer[MAXLEN]; - int n = GetEnvironmentVariableW(envSettingName, buffer, MAXLEN); - - // If none found, check in our two .ini files instead - if (!n) { - n = _readIni(L"defaults", iniSettingName, buffer, MAXLEN); - } - - if (n) { - wchar_t *tag = allocSearchInfoBuffer(search, n + 1); - if (!tag) { - return RC_NO_MEMORY; - } - wcscpy_s(tag, n + 1, buffer); - wchar_t *slash = wcschr(tag, L'/'); - if (!slash) { - search->tag = tag; - search->tagLength = n; - search->oldStyleTag = true; - } else { - search->company = tag; - search->companyLength = (int)(slash - tag); - search->tag = slash + 1; - search->tagLength = n - (search->companyLength + 1); - search->oldStyleTag = false; - } - // gh-92817: allow a high priority env to be selected even if it - // doesn't match the tag - search->lowPriorityTag = true; - } - - return 0; -} - -/******************************************************************************\ - *** ENVIRONMENT SEARCH *** -\******************************************************************************/ - -typedef struct EnvironmentInfo { - /* We use a binary tree and sort on insert */ - struct EnvironmentInfo *prev; - struct EnvironmentInfo *next; - /* parent is only used when constructing */ - struct EnvironmentInfo *parent; - const wchar_t *company; - const wchar_t *tag; - int internalSortKey; - const wchar_t *installDir; - const wchar_t *executablePath; - const wchar_t *executableArgs; - const wchar_t *architecture; - const wchar_t *displayName; - bool highPriority; -} EnvironmentInfo; - - -int -copyWstr(const wchar_t **dest, const wchar_t *src) -{ - if (!dest) { - return RC_NO_MEMORY; - } - if (!src) { - *dest = NULL; - return 0; - } - size_t n = wcsnlen_s(src, MAXLEN - 1) + 1; - wchar_t *buffer = (wchar_t*)malloc(n * sizeof(wchar_t)); - if (!buffer) { - return RC_NO_MEMORY; - } - wcsncpy_s(buffer, n, src, n - 1); - *dest = (const wchar_t*)buffer; - return 0; -} - - -EnvironmentInfo * -newEnvironmentInfo(const wchar_t *company, const wchar_t *tag) -{ - EnvironmentInfo *env = (EnvironmentInfo *)malloc(sizeof(EnvironmentInfo)); - if (!env) { - return NULL; - } - memset(env, 0, sizeof(EnvironmentInfo)); - int exitCode = copyWstr(&env->company, company); - if (exitCode) { - free((void *)env); - return NULL; - } - exitCode = copyWstr(&env->tag, tag); - if (exitCode) { - free((void *)env->company); - free((void *)env); - return NULL; - } - return env; -} - - -void -freeEnvironmentInfo(EnvironmentInfo *env) -{ - if (env) { - free((void *)env->company); - free((void *)env->tag); - free((void *)env->installDir); - free((void *)env->executablePath); - free((void *)env->executableArgs); - free((void *)env->displayName); - freeEnvironmentInfo(env->prev); - env->prev = NULL; - freeEnvironmentInfo(env->next); - env->next = NULL; - free((void *)env); - } -} - - -/* Specific string comparisons for sorting the tree */ - -int -_compareCompany(const wchar_t *x, const wchar_t *y) -{ - if (!x && !y) { - return 0; - } else if (!x) { - return -1; - } else if (!y) { - return 1; - } - - bool coreX = 0 == _compare(x, -1, L"PythonCore", -1); - bool coreY = 0 == _compare(y, -1, L"PythonCore", -1); - if (coreX) { - return coreY ? 0 : -1; - } else if (coreY) { - return 1; - } - return _compare(x, -1, y, -1); -} - - -int -_compareTag(const wchar_t *x, const wchar_t *y) -{ - if (!x && !y) { - return 0; - } else if (!x) { - return -1; - } else if (!y) { - return 1; - } - - // Compare up to the first dash. If not equal, that's our sort order - const wchar_t *xDash = wcschr(x, L'-'); - const wchar_t *yDash = wcschr(y, L'-'); - int xToDash = xDash ? (int)(xDash - x) : -1; - int yToDash = yDash ? (int)(yDash - y) : -1; - int r = _compare(x, xToDash, y, yToDash); - if (r) { - return r; - } - // If we're equal up to the first dash, we want to sort one with - // no dash *after* one with a dash. Otherwise, a reversed compare. - // This works out because environments are sorted in descending tag - // order, so that higher versions (probably) come first. - // For PythonCore, our "X.Y" structure ensures that higher versions - // come first. Everyone else will just have to deal with it. - if (xDash && yDash) { - return _compare(yDash, -1, xDash, -1); - } else if (xDash) { - return -1; - } else if (yDash) { - return 1; - } - return 0; -} - - -int -addEnvironmentInfo(EnvironmentInfo **root, EnvironmentInfo* parent, EnvironmentInfo *node) -{ - EnvironmentInfo *r = *root; - if (!r) { - *root = node; - node->parent = parent; - return 0; - } - // Sort by company name - switch (_compareCompany(node->company, r->company)) { - case -1: - return addEnvironmentInfo(&r->prev, r, node); - case 1: - return addEnvironmentInfo(&r->next, r, node); - case 0: - break; - } - // Then by tag (descending) - switch (_compareTag(node->tag, r->tag)) { - case -1: - return addEnvironmentInfo(&r->next, r, node); - case 1: - return addEnvironmentInfo(&r->prev, r, node); - case 0: - break; - } - // Then keep the one with the lowest internal sort key - if (node->internalSortKey < r->internalSortKey) { - // Replace the current node - node->parent = r->parent; - if (node->parent) { - if (node->parent->prev == r) { - node->parent->prev = node; - } else if (node->parent->next == r) { - node->parent->next = node; - } else { - debug(L"# Inconsistent parent value in tree\n"); - freeEnvironmentInfo(node); - return RC_INTERNAL_ERROR; - } - } else { - // If node has no parent, then it is the root. - *root = node; - } - - node->next = r->next; - node->prev = r->prev; - - debug(L"# replaced %s/%s/%i in tree\n", node->company, node->tag, node->internalSortKey); - freeEnvironmentInfo(r); - } else { - debug(L"# not adding %s/%s/%i to tree\n", node->company, node->tag, node->internalSortKey); - return RC_DUPLICATE_ITEM; - } - return 0; -} - - -/******************************************************************************\ - *** REGISTRY SEARCH *** -\******************************************************************************/ - - -int -_registryReadString(const wchar_t **dest, HKEY root, const wchar_t *subkey, const wchar_t *value) -{ - // Note that this is bytes (hence 'cb'), not characters ('cch') - DWORD cbData = 0; - DWORD flags = RRF_RT_REG_SZ | RRF_RT_REG_EXPAND_SZ; - - if (ERROR_SUCCESS != RegGetValueW(root, subkey, value, flags, NULL, NULL, &cbData)) { - return 0; - } - - wchar_t *buffer = (wchar_t*)malloc(cbData); - if (!buffer) { - return RC_NO_MEMORY; - } - - if (ERROR_SUCCESS == RegGetValueW(root, subkey, value, flags, NULL, buffer, &cbData)) { - *dest = buffer; - } else { - free((void *)buffer); - } - return 0; -} - - -int -_combineWithInstallDir(const wchar_t **dest, const wchar_t *installDir, const wchar_t *fragment, int fragmentLength) -{ - wchar_t buffer[MAXLEN]; - wchar_t fragmentBuffer[MAXLEN]; - if (wcsncpy_s(fragmentBuffer, MAXLEN, fragment, fragmentLength)) { - return RC_NO_MEMORY; - } - - if (FAILED(PathCchCombineEx(buffer, MAXLEN, installDir, fragmentBuffer, PATHCCH_ALLOW_LONG_PATHS))) { - return RC_NO_MEMORY; - } - - return copyWstr(dest, buffer); -} - - -bool -_isLegacyVersion(EnvironmentInfo *env) -{ - // Check if backwards-compatibility is required. - // Specifically PythonCore versions 2.X and 3.0 - 3.5 do not implement PEP 514. - if (0 != _compare(env->company, -1, L"PythonCore", -1)) { - return false; - } - - int versionMajor, versionMinor; - int n = swscanf_s(env->tag, L"%d.%d", &versionMajor, &versionMinor); - if (n != 2) { - debug(L"# %s/%s has an invalid version tag\n", env->company, env->tag); - return false; - } - - return versionMajor == 2 - || (versionMajor == 3 && versionMinor >= 0 && versionMinor <= 5); -} - -int -_registryReadLegacyEnvironment(const SearchInfo *search, HKEY root, EnvironmentInfo *env, const wchar_t *fallbackArch) -{ - // Backwards-compatibility for PythonCore versions which do not implement PEP 514. - int exitCode = _combineWithInstallDir( - &env->executablePath, - env->installDir, - search->executable, - search->executableLength - ); - if (exitCode) { - return exitCode; - } - - if (search->windowed) { - exitCode = _registryReadString(&env->executableArgs, root, L"InstallPath", L"WindowedExecutableArguments"); - } - else { - exitCode = _registryReadString(&env->executableArgs, root, L"InstallPath", L"ExecutableArguments"); - } - if (exitCode) { - return exitCode; - } - - if (fallbackArch) { - copyWstr(&env->architecture, fallbackArch); - } else { - DWORD binaryType; - BOOL success = GetBinaryTypeW(env->executablePath, &binaryType); - if (!success) { - return RC_NO_PYTHON; - } - - switch (binaryType) { - case SCS_32BIT_BINARY: - copyWstr(&env->architecture, L"32bit"); - break; - case SCS_64BIT_BINARY: - copyWstr(&env->architecture, L"64bit"); - break; - default: - return RC_NO_PYTHON; - } - } - - if (0 == _compare(env->architecture, -1, L"32bit", -1)) { - size_t tagLength = wcslen(env->tag); - if (tagLength <= 3 || 0 != _compare(&env->tag[tagLength - 3], 3, L"-32", 3)) { - const wchar_t *rawTag = env->tag; - wchar_t *realTag = (wchar_t*) malloc(sizeof(wchar_t) * (tagLength + 4)); - if (!realTag) { - return RC_NO_MEMORY; - } - - int count = swprintf_s(realTag, tagLength + 4, L"%s-32", env->tag); - if (count == -1) { - debug(L"# Failed to generate 32bit tag\n"); - free(realTag); - return RC_INTERNAL_ERROR; - } - - env->tag = realTag; - free((void*)rawTag); - } - } - - wchar_t buffer[MAXLEN]; - if (swprintf_s(buffer, MAXLEN, L"Python %s", env->tag)) { - copyWstr(&env->displayName, buffer); - } - - return 0; -} - - -int -_registryReadEnvironment(const SearchInfo *search, HKEY root, EnvironmentInfo *env, const wchar_t *fallbackArch) -{ - int exitCode = _registryReadString(&env->installDir, root, L"InstallPath", NULL); - if (exitCode) { - return exitCode; - } - if (!env->installDir) { - return RC_NO_PYTHON; - } - - if (_isLegacyVersion(env)) { - return _registryReadLegacyEnvironment(search, root, env, fallbackArch); - } - - // If pythonw.exe requested, check specific value - if (search->windowed) { - exitCode = _registryReadString(&env->executablePath, root, L"InstallPath", L"WindowedExecutablePath"); - if (!exitCode && env->executablePath) { - exitCode = _registryReadString(&env->executableArgs, root, L"InstallPath", L"WindowedExecutableArguments"); - } - } - if (exitCode) { - return exitCode; - } - - // Missing windowed path or non-windowed request means we use ExecutablePath - if (!env->executablePath) { - exitCode = _registryReadString(&env->executablePath, root, L"InstallPath", L"ExecutablePath"); - if (!exitCode && env->executablePath) { - exitCode = _registryReadString(&env->executableArgs, root, L"InstallPath", L"ExecutableArguments"); - } - } - if (exitCode) { - return exitCode; - } - - if (!env->executablePath) { - debug(L"# %s/%s has no executable path\n", env->company, env->tag); - return RC_NO_PYTHON; - } - - exitCode = _registryReadString(&env->architecture, root, NULL, L"SysArchitecture"); - if (exitCode) { - return exitCode; - } - - exitCode = _registryReadString(&env->displayName, root, NULL, L"DisplayName"); - if (exitCode) { - return exitCode; - } - - return 0; -} - -int -_registrySearchTags(const SearchInfo *search, EnvironmentInfo **result, HKEY root, int sortKey, const wchar_t *company, const wchar_t *fallbackArch) -{ - wchar_t buffer[256]; - int err = 0; - int exitCode = 0; - for (int i = 0; exitCode == 0; ++i) { - DWORD cchBuffer = sizeof(buffer) / sizeof(buffer[0]); - err = RegEnumKeyExW(root, i, buffer, &cchBuffer, NULL, NULL, NULL, NULL); - if (err) { - if (err != ERROR_NO_MORE_ITEMS) { - winerror(0, L"Failed to read installs (tags) from the registry"); - } - break; - } - HKEY subkey; - if (ERROR_SUCCESS == RegOpenKeyExW(root, buffer, 0, KEY_READ, &subkey)) { - EnvironmentInfo *env = newEnvironmentInfo(company, buffer); - env->internalSortKey = sortKey; - exitCode = _registryReadEnvironment(search, subkey, env, fallbackArch); - RegCloseKey(subkey); - if (exitCode == RC_NO_PYTHON) { - freeEnvironmentInfo(env); - exitCode = 0; - } else if (!exitCode) { - exitCode = addEnvironmentInfo(result, NULL, env); - if (exitCode) { - freeEnvironmentInfo(env); - if (exitCode == RC_DUPLICATE_ITEM) { - exitCode = 0; - } - } - } - } - } - return exitCode; -} - - -int -registrySearch(const SearchInfo *search, EnvironmentInfo **result, HKEY root, int sortKey, const wchar_t *fallbackArch) -{ - wchar_t buffer[256]; - int err = 0; - int exitCode = 0; - for (int i = 0; exitCode == 0; ++i) { - DWORD cchBuffer = sizeof(buffer) / sizeof(buffer[0]); - err = RegEnumKeyExW(root, i, buffer, &cchBuffer, NULL, NULL, NULL, NULL); - if (err) { - if (err != ERROR_NO_MORE_ITEMS) { - winerror(0, L"Failed to read distributors (company) from the registry"); - } - break; - } - if (search->limitToCompany && 0 != _compare(search->limitToCompany, -1, buffer, cchBuffer)) { - debug(L"# Skipping %s due to PYLAUNCHER_LIMIT_TO_COMPANY\n", buffer); - continue; - } - HKEY subkey; - if (ERROR_SUCCESS == RegOpenKeyExW(root, buffer, 0, KEY_READ, &subkey)) { - exitCode = _registrySearchTags(search, result, subkey, sortKey, buffer, fallbackArch); - RegCloseKey(subkey); - } - } - return exitCode; -} - - -/******************************************************************************\ - *** APP PACKAGE SEARCH *** -\******************************************************************************/ - -int -appxSearch(const SearchInfo *search, EnvironmentInfo **result, const wchar_t *packageFamilyName, const wchar_t *tag, int sortKey) -{ - wchar_t realTag[32]; - wchar_t buffer[MAXLEN]; - const wchar_t *exeName = search->executable; - if (!exeName || search->allowExecutableOverride) { - exeName = search->windowed ? L"pythonw.exe" : L"python.exe"; - } - - // Failure to get LocalAppData may just mean we're running as a user who - // doesn't have a profile directory. - // In this case, return "not found", but don't fail. - // Chances are they can't launch Store installs anyway. - if (FAILED(SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, buffer))) { - return RC_NO_PYTHON; - } - - if (!join(buffer, MAXLEN, L"Microsoft\\WindowsApps") || - !join(buffer, MAXLEN, packageFamilyName) || - !join(buffer, MAXLEN, exeName)) { - debug(L"# Failed to construct App Execution Alias path\n"); - return RC_INTERNAL_ERROR; - } - - if (INVALID_FILE_ATTRIBUTES == GetFileAttributesW(buffer)) { - return RC_NO_PYTHON; - } - - // Assume packages are native architecture, which means we need to append - // the '-arm64' on ARM64 host. - wcscpy_s(realTag, 32, tag); - if (isARM64Host()) { - wcscat_s(realTag, 32, L"-arm64"); - } - - EnvironmentInfo *env = newEnvironmentInfo(L"PythonCore", realTag); - if (!env) { - return RC_NO_MEMORY; - } - env->internalSortKey = sortKey; - if (isAMD64Host()) { - copyWstr(&env->architecture, L"64bit"); - } else if (isARM64Host()) { - copyWstr(&env->architecture, L"ARM64"); - } - - copyWstr(&env->executablePath, buffer); - - if (swprintf_s(buffer, MAXLEN, L"Python %s (Store)", tag)) { - copyWstr(&env->displayName, buffer); - } - - int exitCode = addEnvironmentInfo(result, NULL, env); - if (exitCode) { - freeEnvironmentInfo(env); - if (exitCode == RC_DUPLICATE_ITEM) { - exitCode = 0; - } - } - - - return exitCode; -} - - -/******************************************************************************\ - *** OVERRIDDEN EXECUTABLE PATH *** -\******************************************************************************/ - - -int -explicitOverrideSearch(const SearchInfo *search, EnvironmentInfo **result) -{ - if (!search->executablePath) { - return 0; - } - - EnvironmentInfo *env = newEnvironmentInfo(NULL, NULL); - if (!env) { - return RC_NO_MEMORY; - } - env->internalSortKey = 10; - int exitCode = copyWstr(&env->executablePath, search->executablePath); - if (exitCode) { - goto abort; - } - exitCode = copyWstr(&env->displayName, L"Explicit override"); - if (exitCode) { - goto abort; - } - exitCode = addEnvironmentInfo(result, NULL, env); - if (exitCode) { - goto abort; - } - return 0; - -abort: - freeEnvironmentInfo(env); - if (exitCode == RC_DUPLICATE_ITEM) { - exitCode = 0; - } - return exitCode; -} - - -/******************************************************************************\ - *** ACTIVE VIRTUAL ENVIRONMENT SEARCH *** -\******************************************************************************/ - -int -virtualenvSearch(const SearchInfo *search, EnvironmentInfo **result) -{ - int exitCode = 0; - EnvironmentInfo *env = NULL; - wchar_t buffer[MAXLEN]; - int n = GetEnvironmentVariableW(L"VIRTUAL_ENV", buffer, MAXLEN); - if (!n || !join(buffer, MAXLEN, L"Scripts") || !join(buffer, MAXLEN, search->executable)) { - return 0; - } - - DWORD attr = GetFileAttributesW(buffer); - if (INVALID_FILE_ATTRIBUTES == attr && search->lowPriorityTag) { - if (!split_parent(buffer, MAXLEN) || !join(buffer, MAXLEN, L"python.exe")) { - return 0; - } - attr = GetFileAttributesW(buffer); - } - - if (INVALID_FILE_ATTRIBUTES == attr) { - debug(L"Python executable %s missing from virtual env\n", buffer); - return 0; - } - - env = newEnvironmentInfo(NULL, NULL); - if (!env) { - return RC_NO_MEMORY; - } - env->highPriority = true; - env->internalSortKey = 20; - exitCode = copyWstr(&env->displayName, L"Active venv"); - if (exitCode) { - goto abort; - } - exitCode = copyWstr(&env->executablePath, buffer); - if (exitCode) { - goto abort; - } - exitCode = addEnvironmentInfo(result, NULL, env); - if (exitCode) { - goto abort; - } - return 0; - -abort: - freeEnvironmentInfo(env); - if (exitCode == RC_DUPLICATE_ITEM) { - return 0; - } - return exitCode; -} - -/******************************************************************************\ - *** COLLECT ENVIRONMENTS *** -\******************************************************************************/ - - -struct RegistrySearchInfo { - // Registry subkey to search - const wchar_t *subkey; - // Registry hive to search - HKEY hive; - // Flags to use when opening the subkey - DWORD flags; - // Internal sort key to select between "identical" environments discovered - // through different methods - int sortKey; - // Fallback value to assume for PythonCore entries missing a SysArchitecture value - const wchar_t *fallbackArch; -}; - - -struct RegistrySearchInfo REGISTRY_SEARCH[] = { - { - L"Software\\Python", - HKEY_CURRENT_USER, - KEY_READ, - 1, - NULL - }, - { - L"Software\\Python", - HKEY_LOCAL_MACHINE, - KEY_READ | KEY_WOW64_64KEY, - 3, - L"64bit" - }, - { - L"Software\\Python", - HKEY_LOCAL_MACHINE, - KEY_READ | KEY_WOW64_32KEY, - 4, - L"32bit" - }, - { NULL, 0, 0, 0, NULL } -}; - - -struct AppxSearchInfo { - // The package family name. Can be found for an installed package using the - // Powershell "Get-AppxPackage" cmdlet - const wchar_t *familyName; - // The tag to treat the installation as - const wchar_t *tag; - // Internal sort key to select between "identical" environments discovered - // through different methods - int sortKey; -}; - - -struct AppxSearchInfo APPX_SEARCH[] = { - // Releases made through the Store - { L"PythonSoftwareFoundation.Python.3.14_qbz5n2kfra8p0", L"3.14", 10 }, - { L"PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0", L"3.13", 10 }, - { L"PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0", L"3.12", 10 }, - { L"PythonSoftwareFoundation.Python.3.11_qbz5n2kfra8p0", L"3.11", 10 }, - { L"PythonSoftwareFoundation.Python.3.10_qbz5n2kfra8p0", L"3.10", 10 }, - { L"PythonSoftwareFoundation.Python.3.9_qbz5n2kfra8p0", L"3.9", 10 }, - { L"PythonSoftwareFoundation.Python.3.8_qbz5n2kfra8p0", L"3.8", 10 }, - - // Side-loadable releases. Note that the publisher ID changes whenever we - // change our code signing certificate subject, so the newer IDs have higher - // priorities (lower sortKey) - { L"PythonSoftwareFoundation.Python.3.14_3847v3x7pw1km", L"3.14", 11 }, - { L"PythonSoftwareFoundation.Python.3.13_3847v3x7pw1km", L"3.13", 11 }, - { L"PythonSoftwareFoundation.Python.3.12_3847v3x7pw1km", L"3.12", 11 }, - { L"PythonSoftwareFoundation.Python.3.11_3847v3x7pw1km", L"3.11", 11 }, - { L"PythonSoftwareFoundation.Python.3.11_hd69rhyc2wevp", L"3.11", 12 }, - { L"PythonSoftwareFoundation.Python.3.10_3847v3x7pw1km", L"3.10", 11 }, - { L"PythonSoftwareFoundation.Python.3.10_hd69rhyc2wevp", L"3.10", 12 }, - { L"PythonSoftwareFoundation.Python.3.9_3847v3x7pw1km", L"3.9", 11 }, - { L"PythonSoftwareFoundation.Python.3.9_hd69rhyc2wevp", L"3.9", 12 }, - { L"PythonSoftwareFoundation.Python.3.8_hd69rhyc2wevp", L"3.8", 12 }, - { NULL, NULL, 0 } -}; - - -int -collectEnvironments(const SearchInfo *search, EnvironmentInfo **result) -{ - int exitCode = 0; - HKEY root; - EnvironmentInfo *env = NULL; - - if (!result) { - debug(L"# collectEnvironments() was passed a NULL result\n"); - return RC_INTERNAL_ERROR; - } - *result = NULL; - - exitCode = explicitOverrideSearch(search, result); - if (exitCode) { - return exitCode; - } - - exitCode = virtualenvSearch(search, result); - if (exitCode) { - return exitCode; - } - - // If we aren't collecting all items to list them, we can exit now. - if (env && !(search->list || search->listPaths)) { - return 0; - } - - for (struct RegistrySearchInfo *info = REGISTRY_SEARCH; info->subkey; ++info) { - if (ERROR_SUCCESS == RegOpenKeyExW(info->hive, info->subkey, 0, info->flags, &root)) { - exitCode = registrySearch(search, result, root, info->sortKey, info->fallbackArch); - RegCloseKey(root); - } - if (exitCode) { - return exitCode; - } - } - - if (search->limitToCompany) { - debug(L"# Skipping APPX search due to PYLAUNCHER_LIMIT_TO_COMPANY\n"); - return 0; - } - - for (struct AppxSearchInfo *info = APPX_SEARCH; info->familyName; ++info) { - exitCode = appxSearch(search, result, info->familyName, info->tag, info->sortKey); - if (exitCode && exitCode != RC_NO_PYTHON) { - return exitCode; - } - } - - return 0; -} - - -/******************************************************************************\ - *** INSTALL ON DEMAND *** -\******************************************************************************/ - -struct StoreSearchInfo { - // The tag a user is looking for - const wchar_t *tag; - // The Store ID for a package if it can be installed from the Microsoft - // Store. These are obtained from the dashboard at - // https://partner.microsoft.com/dashboard - const wchar_t *storeId; -}; - - -struct StoreSearchInfo STORE_SEARCH[] = { - { L"3", /* 3.13 */ L"9PNRBTZXMB4Z" }, - { L"3.14", L"9NTRHQCBBPR8" }, - { L"3.13", L"9PNRBTZXMB4Z" }, - { L"3.12", L"9NCVDN91XZQP" }, - { L"3.11", L"9NRWMJP3717K" }, - { L"3.10", L"9PJPW5LDXLZ5" }, - { L"3.9", L"9P7QFQMJRFP7" }, - { L"3.8", L"9MSSZTT1N39L" }, - { NULL, NULL } -}; - - -int -_installEnvironment(const wchar_t *command, const wchar_t *arguments) -{ - SHELLEXECUTEINFOW siw = { - sizeof(SHELLEXECUTEINFOW), - SEE_MASK_NOASYNC | SEE_MASK_NOCLOSEPROCESS | SEE_MASK_NO_CONSOLE, - NULL, NULL, - command, arguments, NULL, - SW_SHOWNORMAL - }; - - debug(L"# Installing with %s %s\n", command, arguments); - if (isEnvVarSet(L"PYLAUNCHER_DRYRUN")) { - debug(L"# Exiting due to PYLAUNCHER_DRYRUN\n"); - fflush(stdout); - int mode = _setmode(_fileno(stdout), _O_U8TEXT); - if (arguments) { - fwprintf_s(stdout, L"\"%s\" %s\n", command, arguments); - } else { - fwprintf_s(stdout, L"\"%s\"\n", command); - } - fflush(stdout); - if (mode >= 0) { - _setmode(_fileno(stdout), mode); - } - return RC_INSTALLING; - } - - if (!ShellExecuteExW(&siw)) { - return RC_NO_PYTHON; - } - - if (!siw.hProcess) { - return RC_INSTALLING; - } - - WaitForSingleObjectEx(siw.hProcess, INFINITE, FALSE); - DWORD exitCode = 0; - if (GetExitCodeProcess(siw.hProcess, &exitCode) && exitCode == 0) { - return 0; - } - return RC_INSTALLING; -} - - -const wchar_t *WINGET_COMMAND = L"Microsoft\\WindowsApps\\Microsoft.DesktopAppInstaller_8wekyb3d8bbwe\\winget.exe"; -const wchar_t *WINGET_ARGUMENTS = L"install -q %s --exact --accept-package-agreements --source msstore"; - -const wchar_t *MSSTORE_COMMAND = L"ms-windows-store://pdp/?productid=%s"; - -int -installEnvironment(const SearchInfo *search) -{ - // No tag? No installing - if (!search->tag || !search->tagLength) { - debug(L"# Cannot install Python with no tag specified\n"); - return RC_NO_PYTHON; - } - - // PEP 514 tag but not PythonCore? No installing - if (!search->oldStyleTag && - search->company && search->companyLength && - 0 != _compare(search->company, search->companyLength, L"PythonCore", -1)) { - debug(L"# Cannot install for company %.*s\n", search->companyLength, search->company); - return RC_NO_PYTHON; - } - - const wchar_t *storeId = NULL; - for (struct StoreSearchInfo *info = STORE_SEARCH; info->tag; ++info) { - if (0 == _compare(search->tag, search->tagLength, info->tag, -1)) { - storeId = info->storeId; - break; - } - } - - if (!storeId) { - return RC_NO_PYTHON; - } - - int exitCode; - wchar_t command[MAXLEN]; - wchar_t arguments[MAXLEN]; - if (SUCCEEDED(SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, command)) && - join(command, MAXLEN, WINGET_COMMAND) && - swprintf_s(arguments, MAXLEN, WINGET_ARGUMENTS, storeId)) { - if (INVALID_FILE_ATTRIBUTES == GetFileAttributesW(command)) { - formatWinerror(GetLastError(), arguments, MAXLEN); - debug(L"# Skipping %s: %s\n", command, arguments); - } else { - fputws(L"Launching winget to install Python. The following output is from the install process\n\ -***********************************************************************\n", stdout); - exitCode = _installEnvironment(command, arguments); - if (exitCode == RC_INSTALLING) { - fputws(L"***********************************************************************\n\ -Please check the install status and run your command again.", stderr); - return exitCode; - } else if (exitCode) { - return exitCode; - } - fputws(L"***********************************************************************\n\ -Install appears to have succeeded. Searching for new matching installs.\n", stdout); - return 0; - } - } - - if (swprintf_s(command, MAXLEN, MSSTORE_COMMAND, storeId)) { - fputws(L"Opening the Microsoft Store to install Python. After installation, " - L"please run your command again.\n", stderr); - exitCode = _installEnvironment(command, NULL); - if (exitCode) { - return exitCode; - } - return 0; - } - - return RC_NO_PYTHON; -} - -/******************************************************************************\ - *** ENVIRONMENT SELECT *** -\******************************************************************************/ - -bool -_companyMatches(const SearchInfo *search, const EnvironmentInfo *env) -{ - if (!search->company || !search->companyLength) { - return true; - } - return 0 == _compare(env->company, -1, search->company, search->companyLength); -} - - -bool -_tagMatches(const SearchInfo *search, const EnvironmentInfo *env, int searchTagLength) -{ - if (searchTagLength < 0) { - searchTagLength = search->tagLength; - } - if (!search->tag || !searchTagLength) { - return true; - } - return _startsWithSeparated(env->tag, -1, search->tag, searchTagLength, L".-"); -} - - -bool -_is32Bit(const EnvironmentInfo *env) -{ - if (env->architecture) { - return 0 == _compare(env->architecture, -1, L"32bit", -1); - } - return false; -} - - -int -_selectEnvironment(const SearchInfo *search, EnvironmentInfo *env, EnvironmentInfo **best) -{ - int exitCode = 0; - while (env) { - exitCode = _selectEnvironment(search, env->prev, best); - - if (exitCode && exitCode != RC_NO_PYTHON) { - return exitCode; - } else if (!exitCode && *best) { - return 0; - } - - if (env->highPriority && search->lowPriorityTag) { - // This environment is marked high priority, and the search allows - // it to be selected even though a tag is specified, so select it - // gh-92817: this allows an active venv to be selected even when a - // default tag has been found in py.ini or the environment - *best = env; - return 0; - } - - if (!search->oldStyleTag) { - if (_companyMatches(search, env) && _tagMatches(search, env, -1)) { - // Because of how our sort tree is set up, we will walk up the - // "prev" side and implicitly select the "best" best. By - // returning straight after a match, we skip the entire "next" - // branch and won't ever select a "worse" best. - *best = env; - return 0; - } - } else if (0 == _compare(env->company, -1, L"PythonCore", -1)) { - // Old-style tags can only match PythonCore entries - - // If the tag ends with -64, we want to exclude 32-bit runtimes - // (If the tag ends with -32, it will be filtered later) - int tagLength = search->tagLength; - bool exclude32Bit = false, only32Bit = false; - if (tagLength > 3) { - if (0 == _compareArgument(&search->tag[tagLength - 3], 3, L"-64", 3)) { - tagLength -= 3; - exclude32Bit = true; - } else if (0 == _compareArgument(&search->tag[tagLength - 3], 3, L"-32", 3)) { - tagLength -= 3; - only32Bit = true; - } - } - - if (_tagMatches(search, env, tagLength)) { - if (exclude32Bit && _is32Bit(env)) { - debug(L"# Excluding %s/%s because it looks like 32bit\n", env->company, env->tag); - } else if (only32Bit && !_is32Bit(env)) { - debug(L"# Excluding %s/%s because it doesn't look 32bit\n", env->company, env->tag); - } else { - *best = env; - return 0; - } - } - } - - env = env->next; - } - return RC_NO_PYTHON; -} - -int -selectEnvironment(const SearchInfo *search, EnvironmentInfo *root, EnvironmentInfo **best) -{ - if (!best) { - debug(L"# selectEnvironment() was passed a NULL best\n"); - return RC_INTERNAL_ERROR; - } - if (!root) { - *best = NULL; - return RC_NO_PYTHON_AT_ALL; - } - - EnvironmentInfo *result = NULL; - int exitCode = _selectEnvironment(search, root, &result); - if (!exitCode) { - *best = result; - } - - return exitCode; -} - - -/******************************************************************************\ - *** LIST ENVIRONMENTS *** -\******************************************************************************/ - -#define TAGWIDTH 16 - -int -_printEnvironment(const EnvironmentInfo *env, FILE *out, bool showPath, const wchar_t *argument) -{ - if (showPath) { - if (env->executablePath && env->executablePath[0]) { - if (env->executableArgs && env->executableArgs[0]) { - fwprintf(out, L" %-*s %s %s\n", TAGWIDTH, argument, env->executablePath, env->executableArgs); - } else { - fwprintf(out, L" %-*s %s\n", TAGWIDTH, argument, env->executablePath); - } - } else if (env->installDir && env->installDir[0]) { - fwprintf(out, L" %-*s %s\n", TAGWIDTH, argument, env->installDir); - } else { - fwprintf(out, L" %s\n", argument); - } - } else if (env->displayName) { - fwprintf(out, L" %-*s %s\n", TAGWIDTH, argument, env->displayName); - } else { - fwprintf(out, L" %s\n", argument); - } - return 0; -} - - -int -_listAllEnvironments(EnvironmentInfo *env, FILE * out, bool showPath, EnvironmentInfo *defaultEnv) -{ - wchar_t buffer[256]; - const int bufferSize = 256; - while (env) { - int exitCode = _listAllEnvironments(env->prev, out, showPath, defaultEnv); - if (exitCode) { - return exitCode; - } - - if (!env->company || !env->tag) { - buffer[0] = L'\0'; - } else if (0 == _compare(env->company, -1, L"PythonCore", -1)) { - swprintf_s(buffer, bufferSize, L"-V:%s", env->tag); - } else { - swprintf_s(buffer, bufferSize, L"-V:%s/%s", env->company, env->tag); - } - - if (env == defaultEnv) { - wcscat_s(buffer, bufferSize, L" *"); - } - - if (buffer[0]) { - exitCode = _printEnvironment(env, out, showPath, buffer); - if (exitCode) { - return exitCode; - } - } - - env = env->next; - } - return 0; -} - - -int -listEnvironments(EnvironmentInfo *env, FILE * out, bool showPath, EnvironmentInfo *defaultEnv) -{ - if (!env) { - fwprintf_s(stdout, L"No installed Pythons found!\n"); - return 0; - } - - /* TODO: Do we want to display these? - In favour, helps users see that '-3' is a good option - Against, repeats the next line of output - SearchInfo majorSearch; - EnvironmentInfo *major; - int exitCode; - - if (showPath) { - memset(&majorSearch, 0, sizeof(majorSearch)); - majorSearch.company = L"PythonCore"; - majorSearch.companyLength = -1; - majorSearch.tag = L"3"; - majorSearch.tagLength = -1; - majorSearch.oldStyleTag = true; - major = NULL; - exitCode = selectEnvironment(&majorSearch, env, &major); - if (!exitCode && major) { - exitCode = _printEnvironment(major, out, showPath, L"-3 *"); - isDefault = false; - if (exitCode) { - return exitCode; - } - } - majorSearch.tag = L"2"; - major = NULL; - exitCode = selectEnvironment(&majorSearch, env, &major); - if (!exitCode && major) { - exitCode = _printEnvironment(major, out, showPath, L"-2"); - if (exitCode) { - return exitCode; - } - } - } - */ - - int mode = _setmode(_fileno(out), _O_U8TEXT); - int exitCode = _listAllEnvironments(env, out, showPath, defaultEnv); - fflush(out); - if (mode >= 0) { - _setmode(_fileno(out), mode); - } - return exitCode; -} - - -/******************************************************************************\ - *** INTERPRETER LAUNCH *** -\******************************************************************************/ - - -int -calculateCommandLine(const SearchInfo *search, const EnvironmentInfo *launch, wchar_t *buffer, int bufferLength) -{ - int exitCode = 0; - const wchar_t *executablePath = NULL; - - // Construct command line from a search override, or else the selected - // environment's executablePath - if (search->executablePath) { - executablePath = search->executablePath; - } else if (launch && launch->executablePath) { - executablePath = launch->executablePath; - } - - // If we have an executable path, put it at the start of the command, but - // only if the search allowed an override. - // Otherwise, use the environment's installDir and the search's default - // executable name. - if (executablePath && search->allowExecutableOverride) { - if (wcschr(executablePath, L' ') && executablePath[0] != L'"') { - buffer[0] = L'"'; - exitCode = wcscpy_s(&buffer[1], bufferLength - 1, executablePath); - if (!exitCode) { - exitCode = wcscat_s(buffer, bufferLength, L"\""); - } - } else { - exitCode = wcscpy_s(buffer, bufferLength, executablePath); - } - } else if (launch) { - if (!launch->installDir) { - fwprintf_s(stderr, L"Cannot launch %s %s because no install directory was specified", - launch->company, launch->tag); - exitCode = RC_NO_PYTHON; - } else if (!search->executable || !search->executableLength) { - fwprintf_s(stderr, L"Cannot launch %s %s because no executable name is available", - launch->company, launch->tag); - exitCode = RC_NO_PYTHON; - } else { - wchar_t executable[256]; - wcsncpy_s(executable, 256, search->executable, search->executableLength); - if ((wcschr(launch->installDir, L' ') && launch->installDir[0] != L'"') || - (wcschr(executable, L' ') && executable[0] != L'"')) { - buffer[0] = L'"'; - exitCode = wcscpy_s(&buffer[1], bufferLength - 1, launch->installDir); - if (!exitCode) { - exitCode = join(buffer, bufferLength, executable) ? 0 : RC_NO_MEMORY; - } - if (!exitCode) { - exitCode = wcscat_s(buffer, bufferLength, L"\""); - } - } else { - exitCode = wcscpy_s(buffer, bufferLength, launch->installDir); - if (!exitCode) { - exitCode = join(buffer, bufferLength, executable) ? 0 : RC_NO_MEMORY; - } - } - } - } else { - exitCode = RC_NO_PYTHON; - } - - if (!exitCode && launch && launch->executableArgs) { - exitCode = wcscat_s(buffer, bufferLength, L" "); - if (!exitCode) { - exitCode = wcscat_s(buffer, bufferLength, launch->executableArgs); - } - } - - if (!exitCode && search->executableArgs) { - if (search->executableArgsLength < 0) { - exitCode = wcscat_s(buffer, bufferLength, search->executableArgs); - } else if (search->executableArgsLength > 0) { - int end = (int)wcsnlen_s(buffer, MAXLEN); - if (end < bufferLength - (search->executableArgsLength + 1)) { - exitCode = wcsncpy_s(&buffer[end], bufferLength - end, - search->executableArgs, search->executableArgsLength); - } - } - } - - if (!exitCode && search->restOfCmdLine) { - exitCode = wcscat_s(buffer, bufferLength, search->restOfCmdLine); - } - - return exitCode; -} - - - -BOOL -_safeDuplicateHandle(HANDLE in, HANDLE * pout, const wchar_t *nameForError) -{ - BOOL ok; - HANDLE process = GetCurrentProcess(); - DWORD rc; - - *pout = NULL; - ok = DuplicateHandle(process, in, process, pout, 0, TRUE, - DUPLICATE_SAME_ACCESS); - if (!ok) { - rc = GetLastError(); - if (rc == ERROR_INVALID_HANDLE) { - debug(L"DuplicateHandle returned ERROR_INVALID_HANDLE\n"); - ok = TRUE; - } - else { - winerror(0, L"Failed to duplicate %s handle", nameForError); - } - } - return ok; -} - -BOOL WINAPI -ctrl_c_handler(DWORD code) -{ - return TRUE; /* We just ignore all control events. */ -} - - -int -launchEnvironment(const SearchInfo *search, const EnvironmentInfo *launch, wchar_t *launchCommand) -{ - HANDLE job; - JOBOBJECT_EXTENDED_LIMIT_INFORMATION info; - DWORD rc; - BOOL ok; - STARTUPINFOW si; - PROCESS_INFORMATION pi; - - // If this is a dryrun, do not actually launch - if (isEnvVarSet(L"PYLAUNCHER_DRYRUN")) { - debug(L"LaunchCommand: %s\n", launchCommand); - debug(L"# Exiting due to PYLAUNCHER_DRYRUN variable\n"); - fflush(stdout); - int mode = _setmode(_fileno(stdout), _O_U8TEXT); - fwprintf(stdout, L"%s\n", launchCommand); - fflush(stdout); - if (mode >= 0) { - _setmode(_fileno(stdout), mode); - } - return 0; - } - -#if defined(_WINDOWS) - /* - When explorer launches a Windows (GUI) application, it displays - the "app starting" (the "pointer + hourglass") cursor for a number - of seconds, or until the app does something UI-ish (eg, creating a - window, or fetching a message). As this launcher doesn't do this - directly, that cursor remains even after the child process does these - things. We avoid that by doing a simple post+get message. - See http://bugs.python.org/issue17290 - */ - MSG msg; - - PostMessage(0, 0, 0, 0); - GetMessage(&msg, 0, 0, 0); -#endif - - debug(L"# about to run: %s\n", launchCommand); - job = CreateJobObject(NULL, NULL); - ok = QueryInformationJobObject(job, JobObjectExtendedLimitInformation, - &info, sizeof(info), &rc); - if (!ok || (rc != sizeof(info)) || !job) { - winerror(0, L"Failed to query job information"); - return RC_CREATE_PROCESS; - } - info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE | - JOB_OBJECT_LIMIT_SILENT_BREAKAWAY_OK; - ok = SetInformationJobObject(job, JobObjectExtendedLimitInformation, &info, - sizeof(info)); - if (!ok) { - winerror(0, L"Failed to update job information"); - return RC_CREATE_PROCESS; - } - memset(&si, 0, sizeof(si)); - GetStartupInfoW(&si); - if (!_safeDuplicateHandle(GetStdHandle(STD_INPUT_HANDLE), &si.hStdInput, L"stdin") || - !_safeDuplicateHandle(GetStdHandle(STD_OUTPUT_HANDLE), &si.hStdOutput, L"stdout") || - !_safeDuplicateHandle(GetStdHandle(STD_ERROR_HANDLE), &si.hStdError, L"stderr")) { - return RC_NO_STD_HANDLES; - } - - ok = SetConsoleCtrlHandler(ctrl_c_handler, TRUE); - if (!ok) { - winerror(0, L"Failed to update Control-C handler"); - return RC_NO_STD_HANDLES; - } - - si.dwFlags = STARTF_USESTDHANDLES; - ok = CreateProcessW(NULL, launchCommand, NULL, NULL, TRUE, - 0, NULL, NULL, &si, &pi); - if (!ok) { - winerror(0, L"Unable to create process using '%s'", launchCommand); - return RC_CREATE_PROCESS; - } - AssignProcessToJobObject(job, pi.hProcess); - CloseHandle(pi.hThread); - WaitForSingleObjectEx(pi.hProcess, INFINITE, FALSE); - ok = GetExitCodeProcess(pi.hProcess, &rc); - if (!ok) { - winerror(0, L"Failed to get exit code of process"); - return RC_CREATE_PROCESS; - } - debug(L"child process exit code: %d\n", rc); - return rc; -} - - -/******************************************************************************\ - *** PROCESS CONTROLLER *** -\******************************************************************************/ - - -int -performSearch(SearchInfo *search, EnvironmentInfo **envs) -{ - // First parse the command line for options - int exitCode = parseCommandLine(search); - if (exitCode) { - return exitCode; - } - - // Check for a shebang line in our script file - // (or return quickly if no script file was specified) - exitCode = checkShebang(search); - switch (exitCode) { - case 0: - case RC_NO_SHEBANG: - case RC_RECURSIVE_SHEBANG: - break; - case RC_NO_SCRIPT: - if (!_comparePath(search->scriptFile, search->scriptFileLength, L"install", -1) || - !_comparePath(search->scriptFile, search->scriptFileLength, L"uninstall", -1) || - !_comparePath(search->scriptFile, search->scriptFileLength, L"list", -1) || - !_comparePath(search->scriptFile, search->scriptFileLength, L"help", -1)) { - fprintf( - stderr, - "WARNING: The '%.*ls' command is unavailable because this is the legacy py.exe command.\n" - "If you have already installed the Python install manager, open Installed Apps and " - "remove 'Python Launcher' to enable the new py.exe command.\n", - search->scriptFileLength, - search->scriptFile - ); - } - break; - default: - return exitCode; - } - - // Resolve old-style tags (possibly from a shebang) against py.ini entries - // and environment variables. - exitCode = checkDefaults(search); - if (exitCode) { - return exitCode; - } - - // If debugging is enabled, list our search criteria - dumpSearchInfo(search); - - // Find all matching environments - exitCode = collectEnvironments(search, envs); - if (exitCode) { - return exitCode; - } - - return 0; -} - - -int -process(int argc, wchar_t ** argv) -{ - int exitCode = 0; - int searchExitCode = 0; - SearchInfo search = {0}; - EnvironmentInfo *envs = NULL; - EnvironmentInfo *env = NULL; - wchar_t launchCommand[MAXLEN]; - - memset(launchCommand, 0, sizeof(launchCommand)); - - if (isEnvVarSet(L"PYLAUNCHER_DEBUG")) { - setvbuf(stderr, (char *)NULL, _IONBF, 0); - log_fp = stderr; - debug(L"argv0: %s\nversion: %S\n", argv[0], PY_VERSION); - } - - DWORD len = GetEnvironmentVariableW(L"PYLAUNCHER_LIMIT_TO_COMPANY", NULL, 0); - if (len > 1) { - wchar_t *limitToCompany = allocSearchInfoBuffer(&search, len); - if (!limitToCompany) { - exitCode = RC_NO_MEMORY; - winerror(0, L"Failed to allocate internal buffer"); - goto abort; - } - search.limitToCompany = limitToCompany; - if (0 == GetEnvironmentVariableW(L"PYLAUNCHER_LIMIT_TO_COMPANY", limitToCompany, len)) { - exitCode = RC_INTERNAL_ERROR; - winerror(0, L"Failed to read PYLAUNCHER_LIMIT_TO_COMPANY variable"); - goto abort; - } - } - - search.originalCmdLine = GetCommandLineW(); - - exitCode = performSearch(&search, &envs); - if (exitCode) { - goto abort; - } - - // Display the help text, but only exit on error - if (search.help) { - exitCode = showHelpText(argv); - if (exitCode) { - goto abort; - } - } - - // Select best environment - // This is early so that we can show the default when listing, but all - // responses to any errors occur later. - searchExitCode = selectEnvironment(&search, envs, &env); - - // List all environments, then exit - if (search.list || search.listPaths) { - exitCode = listEnvironments(envs, stdout, search.listPaths, env); - goto abort; - } - - // When debugging, list all discovered environments anyway - if (log_fp) { - exitCode = listEnvironments(envs, log_fp, true, NULL); - if (exitCode) { - goto abort; - } - } - - // We searched earlier, so if we didn't find anything, now we react - exitCode = searchExitCode; - // If none found, and if permitted, install it - if (((exitCode == RC_NO_PYTHON) && isEnvVarSet(L"PYLAUNCHER_ALLOW_INSTALL")) || - isEnvVarSet(L"PYLAUNCHER_ALWAYS_INSTALL")) { - exitCode = installEnvironment(&search); - if (!exitCode) { - // Successful install, so we need to re-scan and select again - env = NULL; - exitCode = performSearch(&search, &envs); - if (exitCode) { - goto abort; - } - exitCode = selectEnvironment(&search, envs, &env); - } - } - if (exitCode == RC_NO_PYTHON) { - fputws(L"No suitable Python runtime found\n", stderr); - fputws(L"Pass --list (-0) to see all detected environments on your machine\n", stderr); - if (!isEnvVarSet(L"PYLAUNCHER_ALLOW_INSTALL") && search.oldStyleTag) { - fputws(L"or set environment variable PYLAUNCHER_ALLOW_INSTALL to use winget\n" - L"or open the Microsoft Store to the requested version.\n", stderr); - } - goto abort; - } - if (exitCode == RC_NO_PYTHON_AT_ALL) { - fputws(L"No installed Python found!\n", stderr); - goto abort; - } - if (exitCode) { - goto abort; - } - - if (env) { - debug(L"env.company: %s\nenv.tag: %s\n", env->company, env->tag); - } else { - debug(L"env.company: (null)\nenv.tag: (null)\n"); - } - - exitCode = calculateCommandLine(&search, env, launchCommand, sizeof(launchCommand) / sizeof(launchCommand[0])); - if (exitCode) { - goto abort; - } - - // Launch selected runtime - exitCode = launchEnvironment(&search, env, launchCommand); - -abort: - freeSearchInfo(&search); - freeEnvironmentInfo(envs); - return exitCode; -} - - -#if defined(_WINDOWS) - -int WINAPI wWinMain(HINSTANCE hInstance, HINSTANCE hPrevInstance, - LPWSTR lpstrCmd, int nShow) -{ - return process(__argc, __wargv); -} - -#else - -int cdecl wmain(int argc, wchar_t ** argv) -{ - return process(argc, argv); -} - -#endif diff --git a/PC/layout/README.md b/PC/layout/README.md new file mode 100644 index 000000000000000..086cc23194b1b96 --- /dev/null +++ b/PC/layout/README.md @@ -0,0 +1,341 @@ +# PC/layout script + +This document provides an overview of the `PC/layout` script, which is used to +convert from a build tree to a ready-to-run directory of files. It's the +equivalent of `make install` when using the Makefile. We use it for official +binary releases, but it also supports a lot of configuration and can be used for +whatever your custom scenario should look like. + +The script never builds anything. It reads files from an existing build tree +(`-b`) and the source tree (`-s`), then copies or zips them into a layout. It +does not require the Python being laid out to be runnable, and it can target a +different version from the one running the script (see [Build source +options](#build-source-options)). + +## Quick start + +Run the script as a directory with any Python 3 interpreter. From the +repository root: + +``` +python PC/layout --help +``` + +A useful invocation names an output directory and a preset. The build +directory is inferred from the running interpreter (`-b` overrides it), but a +`--preset-` (or explicit `--include-` options) is required; +otherwise the result is a bare, minimal layout that is rarely useful. + +``` +python PC/layout --copy dist --preset-default +``` + +### Output options + +The script produces one or both of the following outputs. At least one is +required for the run to do anything useful. + +* `--copy ` copies the layout into a directory. +* `--zip ` writes the layout into a ZIP file. + +Additional output-related options: + +* `--catalog ` writes a `.cdf` catalog definition for the non-binary + files, used later to produce a signed catalog. Compilation and signing are + separate steps. +* `--log ` writes all operations to a log file. +* `-t`, `--temp ` sets the temporary working directory used for generated + files (ZIP library, `._pth`, extracted pip, generated JSON). When omitted, a + temporary directory is created automatically. If `--copy` targets a Dev + Drive, an adjacent temp directory on the same drive is used for speed. +* `-v` increases verbosity. Repeat for more detail. + +### Layout options + +Optional components are selected with `--include-` flags. Presets group +these into the configurations we ship (see [Layouts](#layouts)). For a quick +throwaway layout, `-a`/`--include-all` turns on every option (except those +noted below as not affected by `--include-all`); it is intended for debugging +rather than for producing a real distribution, since it mixes components that +are never shipped together. The main component flags are: + +* `--include-stable` the stable ABI DLL (`python3.dll`). +* `--include-pip` pip, extracted from the bundled wheels in + `Lib/ensurepip/_bundled`. +* `--include-pip-user` a `pip.ini` defaulting to `--user` installs. +* `--include-tcltk` Tcl, Tk and tkinter. +* `--include-idle` IDLE. Implies `--include-tcltk`. +* `--include-tests` the test suite. +* `--include-tools` scripts from `Tools`. +* `--include-venv` venv and ensurepip, plus the venv launcher executables. +* `--include-dev` headers and `.lib` files. +* `--include-symbols` `.pdb` symbol files. +* `--include-underpth` a `python._pth` file. Not affected by + `--include-all`. +* `--include-launchers` extra launcher executables (used with + `--include-appxmanifest`). +* `--include-appxmanifest` an APPX/MSIX manifest. +* `--include-props` a `python.props` MSBuild file. +* `--include-nuspec` a `python.nuspec` NuGet spec. +* `--include-chm` the CHM documentation. +* `--include-html-doc` the HTML documentation. +* `--include-freethreaded` free-threaded binaries. Not affected by + `--include-all`. +* `--include-alias`, `--include-alias3`, `--include-alias3x` aliased + entry-point executables (`python.exe`, `python3.exe`, `python3.x.exe` and + their `w` variants). +* `--include-install-json`, `--include-install-embed-json`, + `--include-install-test-json` a PyManager `__install__.json` for the + respective distribution. +* `--include-builddetails-json` a [PEP 739](https://peps.python.org/pep-0739/) + `build-details.json`. + +Layout behavior flags: + +* `-d`, `--debug` lays out the debug build (files with the `_d` suffix). The + debug binaries must already have been built (for example with + `PCbuild/build.bat -d`); the script never builds them. +* `-p`, `--precompile` includes `.pyc` files (at optimization levels 0, 1 and + 2) instead of, or in addition to, `.py` files. The bytecode is compiled + during layout, in-process, using the `py_compile` module of the interpreter + running the script. It does not launch the laid-out runtime and does not check + its version, so it assumes the running interpreter matches the target version; + compiling with a mismatched version produces `.pyc` files the target cannot + use. +* `-z`, `--zip-lib` puts the standard library into `python.zip` rather than + a `Lib` directory. +* `--flat-dlls` places extension modules and DLLs next to the executables + rather than in a `DLLs` directory. +* `--include-cat ` includes an already-built `.cat` catalog file. This + only copies the named file into the layout; it does not build or sign the + catalog and requires no external tooling. Producing the `.cat` from a `.cdf` + (see `--catalog`) is a separate step. + +### Source options + +These options tell the script where to read from and what it is laying out. + +* `-s`, `--source ` the repository root. Defaults to the repository + containing the script. +* `-b`, `--build ` the build directory containing the compiled binaries. + Defaults to the directory containing the running interpreter. +* `--arch ` the target architecture. When omitted, it + is inferred by reading the PE header of a binary in the build directory. +* `--doc-build ` the documentation build directory. Defaults to + `/Doc/build`. + +The following environment variables are read when set: + +* `PYTHON_HEXVERSION` / `PYTHONINCLUDE` override version detection when the + target version cannot be inferred by reading `Include/patchlevel.h` from the + source tree. Set one of these when laying out a version other than the one + running the script (see below). +* `TCL_LIBRARY` locates the Tcl library when `--include-tcltk` is set. If + unset, the script falls back to `TCL_LIBRARY.env` in the build directory. +* `PYTHON_NUSPEC_VERSION`, `PYTHON_PROPS_PLATFORM` fill in the generated + `python.nuspec` and `python.props` files. When unset, the version falls back + to the detected version number and the platform falls back to a value derived + from the target architecture. +* `APPX_DATA_PUBLISHER`, `APPX_DATA_WINVER`, `APPX_DATA_SHA256` fill in the + generated APPX manifest. When unset, they fall back to defaults: a placeholder + publisher certificate name, a `MaxVersionTested` of `10.0.22000.0`, and no + SCCD hash. These defaults are fine for local use but are not suitable for a + signed, Store-ready package. + +## Layouts + +This section lists the common presets, which reflect our supported +configurations. Each is selected with `--preset-` and expands to a fixed +set of `--include-` options. They should help you understand which to use +for your scenario, and they are useful for figuring out where to put a new file +that you've just added to CPython. + +A preset can be combined with additional `--include-` options to add +components on top of it; the options are cumulative. + +### Development Kit (`--preset-default`) + +``` +python PC/layout --copy dist --preset-default +``` + +The development kit layout. This is the closest equivalent to a full install +and contains the interpreter, the standard library in `Lib`, extension modules +in `DLLs`, headers and import libraries (`--include-dev`), symbols, pip, tcltk, +IDLE, venv, the test suite, HTML docs, the stable ABI DLL, aliases and a +`build-details.json`. + +### Python install manager (`--preset-pymanager`, `--preset-pymanager-test`) + +Layouts for the [Python install manager](https://github.com/python/pymanager). +This is now the primary way upstream CPython is distributed on Windows. Both +resemble the development kit but add a PyManager `__install__.json`. + +`pymanager-test` additionally includes the test suite and symbols; +`--preset-pymanager-test` is what produces the `py install PythonTest\` +packages. + +### Embeddable package (`--preset-embed`) + +``` +python PC/layout --zip embed.zip --preset-embed +``` + +The embeddable package: a minimal, self-contained runtime intended to be +bundled inside another application. The standard library is placed in a ZIP +(`--zip-lib`), DLLs are flattened next to the executables (`--flat-dlls`), the +library is precompiled, and a `._pth` file (`--underpth`) is included to fix the +search path and isolate the runtime from user site directories. It omits pip, +tcltk, IDLE, the tests and development files. + +`--preset-embed --include-install-embed-json` is what produces the +`py install PythonEmbed\` packages. + +### NuGet package (`--preset-nuget`) + +The NuGet package layout. Includes development files, pip, the stable ABI, +venv, aliases, a `build-details.json`, and the `python.props` and +`python.nuspec` files that describe the package to MSBuild and NuGet. + +### APPX package (`--preset-appx`) + +The APPX/MSIX package layout for the Windows Store distribution. Includes the +stable ABI, pip, tcltk, IDLE, venv, development files, launchers, aliases and an +APPX manifest. This is no longer used for upstream CPython releases. + +### Windows IoT Core (`--preset-iot`) + +A minimal layout for Windows IoT Core: aliases, the stable ABI and pip. This is +not used for upstream CPython releases. + +## Build source options + +The `-b` and `-s` options let you lay out a build that is separate from the +interpreter running the script, including a different Python version. + +* `-b`/`--build` points at the build tree with the compiled binaries. It does + not need to match the interpreter running the script. +* `-s`/`--source` points at the source tree that supplies `Lib`, `Include`, + `Tools` and version information. + +The target version is normally read from `Include/patchlevel.h` in the source +tree. When laying out a version that differs from the running interpreter and +the source tree is not the matching one, set `PYTHON_HEXVERSION` (a hex version +such as `0x030f00a0`) or `PYTHONINCLUDE` (a directory containing +`patchlevel.h`) so the correct DLL names, suffixes and version numbers are used. +The script validates the inferred version against `patchlevel.h` and stops with +an error if they disagree. + +## Tool structure + +The script is a package under `PC/layout`. `main.py` drives argument parsing, +builds the file list (`get_layout`), generates derived files +(`generate_source_files`) and copies or zips the result (`copy_files`). Support +modules live in `PC/layout/support`: + +* `options.py` defines the `OPTIONS` and `PRESETS` tables and generates the + `--include-*` and `--preset-*` command-line arguments. Add new options and + presets here. +* `constants.py` derives version numbers, DLL names and related constants from + `patchlevel.h` or the version environment variables. +* `filesets.py` provides the file-matching helpers (`FileStemSet`, + `FileNameSet`, `FileSuffixSet`) and the `rglob` globbing helper used + throughout `main.py`. +* `arch.py` infers the target architecture from a binary's PE header. +* `logging.py` configures logging and tracks whether errors occurred. +* `pip.py` extracts pip from the bundled wheels and lists the pip files. +* `catalog.py` generates the `.cdf`/`.cat` catalog definitions for signing. +* `props.py` generates `python.props` (NuGet/MSBuild). +* `nuspec.py` generates `python.nuspec` (NuGet). +* `appxmanifest.py` generates APPX/MSIX manifests. +* `pymanager.py` generates the PyManager `__install__.json`. +* `builddetails.py` generates the PEP 739 `build-details.json`. + +### Classifying new files + +`main.py` collects files by globbing shared directories -- the build output +directory (`-b`) for binaries and the source `Lib`, `Include` and `Tools` +directories -- and then filters them. A new file added to any of these is +usually picked up with no change here: build extension modules (`*.pyd`) land in +`DLLs` (or alongside the executables under `--flat-dlls`), other build DLLs land +next to them, and files under `Lib` are mirrored into the layout's `Lib`. + +Changes are only needed when a new file belongs to an optional component, so it +must be routed to the right `--include-` option rather than always +copied. The routing is driven by a set of file-matching tables near the top of +`main.py` (built from the `FileStemSet`, `FileNameSet` and `FileSuffixSet` +helpers in `support/filesets.py`). To classify a new file, add its name or +pattern to the matching table. + +Tables that gate files on an optional component (the file is only included when +the corresponding option is set): + +* `TEST_PYDS_ONLY`, `TEST_DLLS_ONLY`, `TEST_DIRS_ONLY` -- test-only extension + modules, DLLs and `Lib` directories; gated on `--include-tests`. Extension + module names starting with `_test` are already matched; add any other + test-only build module here (see also the note in `PCbuild/readme.txt`). +* `TCLTK_PYDS_ONLY`, `TCLTK_DLLS_ONLY`, `TCLTK_DIRS_ONLY`, `TCLTK_FILES_ONLY` -- + Tcl/Tk/tkinter extension modules (`_tkinter`), DLLs (`tcl*`, `tk*`, `zlib1`), + `Lib` directories (`tkinter`, `turtledemo`) and files (`turtle.py`); gated on + `--include-tcltk`. +* `IDLE_DIRS_ONLY` -- IDLE's `Lib` directory (`idlelib`); gated on + `--include-idle`. +* `VENV_DIRS_ONLY` -- `venv` and `ensurepip` `Lib` directories; gated on + `--include-venv`. + +Tables that exclude files outright: + +* `EXCLUDE_FROM_DLLS` -- build DLLs that are handled elsewhere and must not be + copied by the generic DLL glob (`python*`, `pyshellext`, `vcruntime*`). +* `EXCLUDE_FROM_LIB` -- never copied into `Lib` (`*.pyc`, `__pycache__`, + `*.pickle`). +* `EXCLUDE_FROM_COMPILE` -- `Lib` files that must not be precompiled + (`badsyntax_*`, `bad_*`). +* `EXCLUDE_FROM_CATALOG` -- suffixes omitted from the signing catalog because + they are signed separately (`.exe`, `.pyd`, `.dll`). + +Special cases in the debug and binary handling: + +* `REQUIRED_DLLS` (`libcrypto*`, `libssl*`, `libffi*`) -- always copied from the + build directory even for a release layout. Most build DLLs are filtered by + their `_d` suffix to match the debug or release build being laid out; these + dependencies do not follow that naming and would otherwise be dropped, so they + are matched here to force inclusion. Add a new external dependency DLL here if + it does not use the `_d` debug-suffix convention. + +When deciding where a new file goes, match it to the component it belongs to and +add it to the corresponding table. If it belongs in every layout and follows the +normal conventions, no change is needed. + + +## Contributing guidelines + +Contributions are welcome. The script supports a broad range of scenarios, and +extending it for a new one is encouraged. Because the same code produces the +official releases across every supported version and architecture, additions +have to respect a few constraints so that existing configurations keep working. + +The script runs in an unusual context, and new code must hold to the following +assumptions: + +* Do not assume the Python being laid out can be launched. The script must run + correctly using only the interpreter running it, which may differ from the + target. +* Do not assume the layout version matches the version running the script. + Derive version-dependent behavior from the detected version rather than from + `sys.version` of the running interpreter. (Precompilation with `--precompile` + is the one deliberate exception, and it assumes a matching interpreter.) +* The script may be used to lay out an older version, so behavior changes have + to be preserved, either behind an option or gated on a version check, rather + than replaced outright. +* The script never builds anything. Everything it emits must already exist in + the build or source tree. + +When adding features, keep the surface area aligned with upstream: + +* New `--include-` options for further customization are fine. +* Do not add presets beyond those used by upstream. Presets describe the + configurations we ship, so a new one implies a new supported release + artifact. +* Route new files through the classification tables (see [Classifying new + files](#classifying-new-files)) rather than adding one-off special cases. diff --git a/PC/layout/support/appxmanifest.py b/PC/layout/support/appxmanifest.py index 53977beb8af8346..ba7f1335c7348d5 100644 --- a/PC/layout/support/appxmanifest.py +++ b/PC/layout/support/appxmanifest.py @@ -151,6 +151,8 @@ """ +# This file is no longer included in our repo, but we keep the logic here for +# building with older releases. We always .is_file() before using it. SCCD_FILENAME = "PC/classicAppCompat.sccd" SPECIAL_LOOKUP = object() diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 8826e7e85a7f5ab..02f16d41b1457b1 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -317,6 +317,22 @@ msvcrt_getwche_impl(PyObject *module) #endif /* MS_WINDOWS_DESKTOP */ +/* Raise an OSError for a failed _putch()/_putwch() call. + + These functions fail, for example, when the process has no console + attached, but the CRT reports the failure without setting errno (and + without setting the Windows last error either), so fall back to a + generic error message in that case. */ +static PyObject * +set_console_write_error(void) +{ + if (errno != 0) { + return PyErr_SetFromErrno(PyExc_OSError); + } + PyErr_SetString(PyExc_OSError, "write to console failed"); + return NULL; +} + /*[clinic input] msvcrt.putch @@ -330,9 +346,16 @@ static PyObject * msvcrt_putch_impl(PyObject *module, char char_value) /*[clinic end generated code: output=92ec9b81012d8f60 input=ec078dd10cb054d6]*/ { + int res; + _Py_BEGIN_SUPPRESS_IPH - _putch(char_value); + errno = 0; + res = _putch(char_value); _Py_END_SUPPRESS_IPH + + if (res == EOF) { + return set_console_write_error(); + } Py_RETURN_NONE; } @@ -351,11 +374,17 @@ static PyObject * msvcrt_putwch_impl(PyObject *module, int unicode_char) /*[clinic end generated code: output=a3bd1a8951d28eee input=996ccd0bbcbac4c3]*/ { + wint_t res; + _Py_BEGIN_SUPPRESS_IPH - _putwch(unicode_char); + errno = 0; + res = _putwch(unicode_char); _Py_END_SUPPRESS_IPH - Py_RETURN_NONE; + if (res == WEOF) { + return set_console_write_error(); + } + Py_RETURN_NONE; } #endif /* MS_WINDOWS_DESKTOP */ diff --git a/PC/pylauncher.rc b/PC/pylauncher.rc deleted file mode 100644 index 11862643aa69893..000000000000000 --- a/PC/pylauncher.rc +++ /dev/null @@ -1,67 +0,0 @@ -#include - -#include "python_ver_rc.h" - -#ifndef RT_MANIFEST -// bpo-45220: Cannot reliably #include RT_MANIFEST from -// anywhere, so we hardcode it -#define RT_MANIFEST 24 -#endif -// Include the manifest file that indicates we support all -// current versions of Windows. -1 RT_MANIFEST "python.manifest" - -#if defined(PY_ICON) -1 ICON DISCARDABLE "icons\python.ico" -#elif defined(PYW_ICON) -1 ICON DISCARDABLE "icons\pythonw.ico" -#else -1 ICON DISCARDABLE "icons\launcher.ico" -2 ICON DISCARDABLE "icons\py.ico" -3 ICON DISCARDABLE "icons\pyc.ico" -4 ICON DISCARDABLE "icons\pyd.ico" -5 ICON DISCARDABLE "icons\python.ico" -6 ICON DISCARDABLE "icons\pythonw.ico" -7 ICON DISCARDABLE "icons\setup.ico" -#endif - -1 USAGE "launcher-usage.txt" - - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION PYVERSION64 - PRODUCTVERSION PYVERSION64 - FILEFLAGSMASK 0x3fL -#ifdef _DEBUG - FILEFLAGS VS_FF_DEBUG -#else - FILEFLAGS 0x0L -#endif - FILEOS VOS__WINDOWS32 - FILETYPE VFT_APP - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "000004b0" - BEGIN - VALUE "CompanyName", PYTHON_COMPANY "\0" - VALUE "FileDescription", "Python\0" - VALUE "FileVersion", PYTHON_VERSION - VALUE "InternalName", "Python Launcher\0" - VALUE "LegalCopyright", PYTHON_COPYRIGHT "\0" - VALUE "OriginalFilename", "py" PYTHON_DEBUG_EXT ".exe\0" - VALUE "ProductName", "Python\0" - VALUE "ProductVersion", PYTHON_VERSION - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x0, 1200 - END -END \ No newline at end of file diff --git a/PC/pyshellext.cpp b/PC/pyshellext.cpp deleted file mode 100644 index d16de0b717151e5..000000000000000 --- a/PC/pyshellext.cpp +++ /dev/null @@ -1,603 +0,0 @@ -// Support back to Vista -#define _WIN32_WINNT _WIN32_WINNT_VISTA -#include - -// Use WRL to define a classic COM class -#define __WRL_CLASSIC_COM__ -#include - -#include -#include -#include -#include -#include - -#define DDWM_UPDATEWINDOW (WM_USER+3) - -static HINSTANCE hModule; -static CLIPFORMAT cfDropDescription; -static CLIPFORMAT cfDragWindow; - -#define CLASS_GUID "{BEA218D2-6950-497B-9434-61683EC065FE}" -static const LPCWSTR CLASS_SUBKEY = L"Software\\Classes\\CLSID\\" CLASS_GUID; -static const LPCWSTR DRAG_MESSAGE = L"Open with %1"; - -using namespace Microsoft::WRL; - -HRESULT FilenameListCchLengthA(LPCSTR pszSource, size_t cchMax, size_t *pcchLength, size_t *pcchCount) { - HRESULT hr = S_OK; - size_t count = 0; - size_t length = 0; - - while (pszSource && pszSource[0]) { - size_t oneLength; - hr = StringCchLengthA(pszSource, cchMax - length, &oneLength); - if (FAILED(hr)) { - return hr; - } - count += 1; - length += oneLength + (strchr(pszSource, ' ') ? 3 : 1); - pszSource = &pszSource[oneLength + 1]; - } - - *pcchCount = count; - *pcchLength = length; - return hr; -} - -HRESULT FilenameListCchLengthW(LPCWSTR pszSource, size_t cchMax, size_t *pcchLength, size_t *pcchCount) { - HRESULT hr = S_OK; - size_t count = 0; - size_t length = 0; - - while (pszSource && pszSource[0]) { - size_t oneLength; - hr = StringCchLengthW(pszSource, cchMax - length, &oneLength); - if (FAILED(hr)) { - return hr; - } - count += 1; - length += oneLength + (wcschr(pszSource, ' ') ? 3 : 1); - pszSource = &pszSource[oneLength + 1]; - } - - *pcchCount = count; - *pcchLength = length; - return hr; -} - -HRESULT FilenameListCchCopyA(STRSAFE_LPSTR pszDest, size_t cchDest, LPCSTR pszSource, LPCSTR pszSeparator) { - HRESULT hr = S_OK; - - while (pszSource[0]) { - STRSAFE_LPSTR newDest; - - hr = StringCchCopyExA(pszDest, cchDest, pszSource, &newDest, &cchDest, 0); - if (FAILED(hr)) { - return hr; - } - pszSource += (newDest - pszDest) + 1; - pszDest = PathQuoteSpacesA(pszDest) ? newDest + 2 : newDest; - - if (pszSource[0]) { - hr = StringCchCopyExA(pszDest, cchDest, pszSeparator, &newDest, &cchDest, 0); - if (FAILED(hr)) { - return hr; - } - pszDest = newDest; - } - } - - return hr; -} - -HRESULT FilenameListCchCopyW(STRSAFE_LPWSTR pszDest, size_t cchDest, LPCWSTR pszSource, LPCWSTR pszSeparator) { - HRESULT hr = S_OK; - - while (pszSource[0]) { - STRSAFE_LPWSTR newDest; - - hr = StringCchCopyExW(pszDest, cchDest, pszSource, &newDest, &cchDest, 0); - if (FAILED(hr)) { - return hr; - } - pszSource += (newDest - pszDest) + 1; - pszDest = PathQuoteSpacesW(pszDest) ? newDest + 2 : newDest; - - if (pszSource[0]) { - hr = StringCchCopyExW(pszDest, cchDest, pszSeparator, &newDest, &cchDest, 0); - if (FAILED(hr)) { - return hr; - } - pszDest = newDest; - } - } - - return hr; -} - -class DECLSPEC_UUID(CLASS_GUID) PyShellExt : public RuntimeClass< - RuntimeClassFlags, - IDropTarget, - IPersistFile -> -{ - LPOLESTR target, target_dir; - DWORD target_mode; - - IDataObject *data_obj; - -public: - PyShellExt() : target(NULL), target_dir(NULL), target_mode(0), data_obj(NULL) { - OutputDebugString(L"PyShellExt::PyShellExt"); - } - - ~PyShellExt() { - if (target) { - CoTaskMemFree(target); - } - if (target_dir) { - CoTaskMemFree(target_dir); - } - if (data_obj) { - data_obj->Release(); - } - } - -private: - HRESULT UpdateDropDescription(IDataObject *pDataObj) { - STGMEDIUM medium; - FORMATETC fmt = { - cfDropDescription, - NULL, - DVASPECT_CONTENT, - -1, - TYMED_HGLOBAL - }; - - auto hr = pDataObj->GetData(&fmt, &medium); - if (FAILED(hr)) { - OutputDebugString(L"PyShellExt::UpdateDropDescription - failed to get DROPDESCRIPTION format"); - return hr; - } - if (!medium.hGlobal) { - OutputDebugString(L"PyShellExt::UpdateDropDescription - DROPDESCRIPTION format had NULL hGlobal"); - ReleaseStgMedium(&medium); - return E_FAIL; - } - auto dd = (DROPDESCRIPTION*)GlobalLock(medium.hGlobal); - if (!dd) { - OutputDebugString(L"PyShellExt::UpdateDropDescription - failed to lock DROPDESCRIPTION hGlobal"); - ReleaseStgMedium(&medium); - return E_FAIL; - } - StringCchCopy(dd->szMessage, sizeof(dd->szMessage) / sizeof(dd->szMessage[0]), DRAG_MESSAGE); - StringCchCopy(dd->szInsert, sizeof(dd->szInsert) / sizeof(dd->szInsert[0]), PathFindFileNameW(target)); - dd->type = DROPIMAGE_MOVE; - - GlobalUnlock(medium.hGlobal); - ReleaseStgMedium(&medium); - - return S_OK; - } - - HRESULT GetDragWindow(IDataObject *pDataObj, HWND *phWnd) { - HRESULT hr; - HWND *pMem; - STGMEDIUM medium; - FORMATETC fmt = { - cfDragWindow, - NULL, - DVASPECT_CONTENT, - -1, - TYMED_HGLOBAL - }; - - hr = pDataObj->GetData(&fmt, &medium); - if (FAILED(hr)) { - OutputDebugString(L"PyShellExt::GetDragWindow - failed to get DragWindow format"); - return hr; - } - if (!medium.hGlobal) { - OutputDebugString(L"PyShellExt::GetDragWindow - DragWindow format had NULL hGlobal"); - ReleaseStgMedium(&medium); - return E_FAIL; - } - - pMem = (HWND*)GlobalLock(medium.hGlobal); - if (!pMem) { - OutputDebugString(L"PyShellExt::GetDragWindow - failed to lock DragWindow hGlobal"); - ReleaseStgMedium(&medium); - return E_FAIL; - } - - *phWnd = *pMem; - - GlobalUnlock(medium.hGlobal); - ReleaseStgMedium(&medium); - - return S_OK; - } - - HRESULT GetArguments(IDataObject *pDataObj, LPCWSTR *pArguments) { - HRESULT hr; - DROPFILES *pdropfiles; - - STGMEDIUM medium; - FORMATETC fmt = { - CF_HDROP, - NULL, - DVASPECT_CONTENT, - -1, - TYMED_HGLOBAL - }; - - hr = pDataObj->GetData(&fmt, &medium); - if (FAILED(hr)) { - OutputDebugString(L"PyShellExt::GetArguments - failed to get CF_HDROP format"); - return hr; - } - if (!medium.hGlobal) { - OutputDebugString(L"PyShellExt::GetArguments - CF_HDROP format had NULL hGlobal"); - ReleaseStgMedium(&medium); - return E_FAIL; - } - - pdropfiles = (DROPFILES*)GlobalLock(medium.hGlobal); - if (!pdropfiles) { - OutputDebugString(L"PyShellExt::GetArguments - failed to lock CF_HDROP hGlobal"); - ReleaseStgMedium(&medium); - return E_FAIL; - } - - if (pdropfiles->fWide) { - LPCWSTR files = (LPCWSTR)((char*)pdropfiles + pdropfiles->pFiles); - size_t len, count; - hr = FilenameListCchLengthW(files, 32767, &len, &count); - if (SUCCEEDED(hr)) { - LPWSTR args = (LPWSTR)CoTaskMemAlloc(sizeof(WCHAR) * (len + 1)); - if (args) { - hr = FilenameListCchCopyW(args, 32767, files, L" "); - if (SUCCEEDED(hr)) { - *pArguments = args; - } else { - CoTaskMemFree(args); - } - } else { - hr = E_OUTOFMEMORY; - } - } - } else { - LPCSTR files = (LPCSTR)((char*)pdropfiles + pdropfiles->pFiles); - size_t len, count; - hr = FilenameListCchLengthA(files, 32767, &len, &count); - if (SUCCEEDED(hr)) { - LPSTR temp = (LPSTR)CoTaskMemAlloc(sizeof(CHAR) * (len + 1)); - if (temp) { - hr = FilenameListCchCopyA(temp, 32767, files, " "); - if (SUCCEEDED(hr)) { - int wlen = MultiByteToWideChar(CP_ACP, 0, temp, (int)len, NULL, 0); - if (wlen) { - LPWSTR args = (LPWSTR)CoTaskMemAlloc(sizeof(WCHAR) * (wlen + 1)); - if (MultiByteToWideChar(CP_ACP, 0, temp, (int)len, args, wlen + 1)) { - *pArguments = args; - } else { - OutputDebugString(L"PyShellExt::GetArguments - failed to convert multi-byte to wide-char path"); - CoTaskMemFree(args); - hr = E_FAIL; - } - } else { - OutputDebugString(L"PyShellExt::GetArguments - failed to get length of wide-char path"); - hr = E_FAIL; - } - } - CoTaskMemFree(temp); - } else { - hr = E_OUTOFMEMORY; - } - } - } - - GlobalUnlock(medium.hGlobal); - ReleaseStgMedium(&medium); - - return hr; - } - - HRESULT NotifyDragWindow(HWND hwnd) { - LRESULT res; - - if (!hwnd) { - return S_FALSE; - } - - res = SendMessage(hwnd, DDWM_UPDATEWINDOW, 0, NULL); - - if (res) { - OutputDebugString(L"PyShellExt::NotifyDragWindow - failed to post DDWM_UPDATEWINDOW"); - return E_FAIL; - } - - return S_OK; - } - -public: - // IDropTarget implementation - - STDMETHODIMP DragEnter(IDataObject *pDataObj, DWORD grfKeyState, POINTL pt, DWORD *pdwEffect) { - HWND hwnd; - - OutputDebugString(L"PyShellExt::DragEnter"); - - pDataObj->AddRef(); - data_obj = pDataObj; - - *pdwEffect = DROPEFFECT_MOVE; - - if (FAILED(UpdateDropDescription(data_obj))) { - OutputDebugString(L"PyShellExt::DragEnter - failed to update drop description"); - } - if (FAILED(GetDragWindow(data_obj, &hwnd))) { - OutputDebugString(L"PyShellExt::DragEnter - failed to get drag window"); - } - if (FAILED(NotifyDragWindow(hwnd))) { - OutputDebugString(L"PyShellExt::DragEnter - failed to notify drag window"); - } - - return S_OK; - } - - STDMETHODIMP DragLeave() { - return S_OK; - } - - STDMETHODIMP DragOver(DWORD grfKeyState, POINTL pt, DWORD *pdwEffect) { - return S_OK; - } - - STDMETHODIMP Drop(IDataObject *pDataObj, DWORD grfKeyState, POINTL pt, DWORD *pdwEffect) { - LPCWSTR args; - - OutputDebugString(L"PyShellExt::Drop"); - *pdwEffect = DROPEFFECT_NONE; - - if (pDataObj != data_obj) { - OutputDebugString(L"PyShellExt::Drop - unexpected data object"); - return E_FAIL; - } - - data_obj->Release(); - data_obj = NULL; - - if (SUCCEEDED(GetArguments(pDataObj, &args))) { - OutputDebugString(args); - ShellExecute(NULL, NULL, target, args, target_dir, SW_NORMAL); - - CoTaskMemFree((LPVOID)args); - } else { - OutputDebugString(L"PyShellExt::Drop - failed to get launch arguments"); - } - - return S_OK; - } - - // IPersistFile implementation - - STDMETHODIMP GetCurFile(LPOLESTR *ppszFileName) { - HRESULT hr; - size_t len; - - if (!ppszFileName) { - return E_POINTER; - } - - hr = StringCchLength(target, STRSAFE_MAX_CCH - 1, &len); - if (FAILED(hr)) { - return E_FAIL; - } - - *ppszFileName = (LPOLESTR)CoTaskMemAlloc(sizeof(WCHAR) * (len + 1)); - if (!*ppszFileName) { - return E_OUTOFMEMORY; - } - - hr = StringCchCopy(*ppszFileName, len + 1, target); - if (FAILED(hr)) { - CoTaskMemFree(*ppszFileName); - *ppszFileName = NULL; - return E_FAIL; - } - - return S_OK; - } - - STDMETHODIMP IsDirty() { - return S_FALSE; - } - - STDMETHODIMP Load(LPCOLESTR pszFileName, DWORD dwMode) { - HRESULT hr; - size_t len; - - OutputDebugString(L"PyShellExt::Load"); - OutputDebugString(pszFileName); - - hr = StringCchLength(pszFileName, STRSAFE_MAX_CCH - 1, &len); - if (FAILED(hr)) { - OutputDebugString(L"PyShellExt::Load - failed to get string length"); - return hr; - } - - if (target) { - CoTaskMemFree(target); - } - if (target_dir) { - CoTaskMemFree(target_dir); - } - - target = (LPOLESTR)CoTaskMemAlloc(sizeof(WCHAR) * (len + 1)); - if (!target) { - OutputDebugString(L"PyShellExt::Load - E_OUTOFMEMORY"); - return E_OUTOFMEMORY; - } - target_dir = (LPOLESTR)CoTaskMemAlloc(sizeof(WCHAR) * (len + 1)); - if (!target_dir) { - OutputDebugString(L"PyShellExt::Load - E_OUTOFMEMORY"); - return E_OUTOFMEMORY; - } - - hr = StringCchCopy(target, len + 1, pszFileName); - if (FAILED(hr)) { - OutputDebugString(L"PyShellExt::Load - failed to copy string"); - return hr; - } - - hr = StringCchCopy(target_dir, len + 1, pszFileName); - if (FAILED(hr)) { - OutputDebugString(L"PyShellExt::Load - failed to copy string"); - return hr; - } - if (!PathRemoveFileSpecW(target_dir)) { - OutputDebugStringW(L"PyShellExt::Load - failed to remove filespec from target"); - return E_FAIL; - } - - OutputDebugString(target); - target_mode = dwMode; - OutputDebugString(L"PyShellExt::Load - S_OK"); - return S_OK; - } - - STDMETHODIMP Save(LPCOLESTR pszFileName, BOOL fRemember) { - return E_NOTIMPL; - } - - STDMETHODIMP SaveCompleted(LPCOLESTR pszFileName) { - return E_NOTIMPL; - } - - STDMETHODIMP GetClassID(CLSID *pClassID) { - *pClassID = __uuidof(PyShellExt); - return S_OK; - } -}; - -CoCreatableClass(PyShellExt); - -STDAPI DllGetClassObject(REFCLSID rclsid, REFIID riid, _COM_Outptr_ void** ppv) { - return Module::GetModule().GetClassObject(rclsid, riid, ppv); -} - -STDAPI DllCanUnloadNow() { - return Module::GetModule().Terminate() ? S_OK : S_FALSE; -} - -STDAPI DllRegisterServer() { - LONG res; - LPSECURITY_ATTRIBUTES psecattr = NULL; - HKEY key, ipsKey; - WCHAR modname[MAX_PATH]; - DWORD modname_len; - - OutputDebugString(L"PyShellExt::DllRegisterServer"); - if (!hModule) { - OutputDebugString(L"PyShellExt::DllRegisterServer - module handle was not set"); - return SELFREG_E_CLASS; - } - modname_len = GetModuleFileName(hModule, modname, MAX_PATH); - if (modname_len == 0 || - (modname_len == MAX_PATH && GetLastError() == ERROR_INSUFFICIENT_BUFFER)) { - OutputDebugString(L"PyShellExt::DllRegisterServer - failed to get module file name"); - return SELFREG_E_CLASS; - } - - DWORD disp; - res = RegCreateKeyEx(HKEY_LOCAL_MACHINE, CLASS_SUBKEY, 0, NULL, 0, - KEY_ALL_ACCESS, psecattr, &key, &disp); - if (res == ERROR_ACCESS_DENIED) { - OutputDebugString(L"PyShellExt::DllRegisterServer - failed to write per-machine registration. Attempting per-user instead."); - res = RegCreateKeyEx(HKEY_CURRENT_USER, CLASS_SUBKEY, 0, NULL, 0, - KEY_ALL_ACCESS, psecattr, &key, &disp); - } - if (res != ERROR_SUCCESS) { - OutputDebugString(L"PyShellExt::DllRegisterServer - failed to create class key"); - return SELFREG_E_CLASS; - } - - res = RegCreateKeyEx(key, L"InProcServer32", 0, NULL, 0, - KEY_ALL_ACCESS, psecattr, &ipsKey, NULL); - if (res != ERROR_SUCCESS) { - RegCloseKey(key); - OutputDebugString(L"PyShellExt::DllRegisterServer - failed to create InProcServer32 key"); - return SELFREG_E_CLASS; - } - - res = RegSetValueEx(ipsKey, NULL, 0, - REG_SZ, (LPBYTE)modname, modname_len * sizeof(modname[0])); - - if (res != ERROR_SUCCESS) { - RegCloseKey(ipsKey); - RegCloseKey(key); - OutputDebugString(L"PyShellExt::DllRegisterServer - failed to set server path"); - return SELFREG_E_CLASS; - } - - res = RegSetValueEx(ipsKey, L"ThreadingModel", 0, - REG_SZ, (LPBYTE)(L"Apartment"), sizeof(L"Apartment")); - - RegCloseKey(ipsKey); - RegCloseKey(key); - if (res != ERROR_SUCCESS) { - OutputDebugString(L"PyShellExt::DllRegisterServer - failed to set threading model"); - return SELFREG_E_CLASS; - } - - SHChangeNotify(SHCNE_ASSOCCHANGED, SHCNF_IDLIST, NULL, NULL); - - OutputDebugString(L"PyShellExt::DllRegisterServer - S_OK"); - return S_OK; -} - -STDAPI DllUnregisterServer() { - LONG res_lm, res_cu; - - res_lm = RegDeleteTree(HKEY_LOCAL_MACHINE, CLASS_SUBKEY); - if (res_lm != ERROR_SUCCESS && res_lm != ERROR_FILE_NOT_FOUND) { - OutputDebugString(L"PyShellExt::DllUnregisterServer - failed to delete per-machine registration"); - return SELFREG_E_CLASS; - } - - res_cu = RegDeleteTree(HKEY_CURRENT_USER, CLASS_SUBKEY); - if (res_cu != ERROR_SUCCESS && res_cu != ERROR_FILE_NOT_FOUND) { - OutputDebugString(L"PyShellExt::DllUnregisterServer - failed to delete per-user registration"); - return SELFREG_E_CLASS; - } - - if (res_lm == ERROR_FILE_NOT_FOUND && res_cu == ERROR_FILE_NOT_FOUND) { - OutputDebugString(L"PyShellExt::DllUnregisterServer - extension was not registered"); - return SELFREG_E_CLASS; - } - - SHChangeNotify(SHCNE_ASSOCCHANGED, SHCNF_IDLIST, NULL, NULL); - - OutputDebugString(L"PyShellExt::DllUnregisterServer - S_OK"); - return S_OK; -} - -STDAPI_(BOOL) DllMain(_In_opt_ HINSTANCE hinst, DWORD reason, _In_opt_ void*) { - if (reason == DLL_PROCESS_ATTACH) { - hModule = hinst; - - cfDropDescription = RegisterClipboardFormat(CFSTR_DROPDESCRIPTION); - if (!cfDropDescription) { - OutputDebugString(L"PyShellExt::DllMain - failed to get CFSTR_DROPDESCRIPTION format"); - } - cfDragWindow = RegisterClipboardFormat(L"DragWindow"); - if (!cfDragWindow) { - OutputDebugString(L"PyShellExt::DllMain - failed to get DragWindow format"); - } - - DisableThreadLibraryCalls(hinst); - } - return TRUE; -} diff --git a/PC/pyshellext.def b/PC/pyshellext.def deleted file mode 100644 index 288a9adf982f197..000000000000000 --- a/PC/pyshellext.def +++ /dev/null @@ -1,5 +0,0 @@ -EXPORTS - DllRegisterServer PRIVATE - DllUnregisterServer PRIVATE - DllGetClassObject PRIVATE - DllCanUnloadNow PRIVATE diff --git a/PC/pyshellext.rc b/PC/pyshellext.rc deleted file mode 100644 index af797ce95d5077e..000000000000000 --- a/PC/pyshellext.rc +++ /dev/null @@ -1,51 +0,0 @@ -#include - -#include "python_ver_rc.h" - -#ifndef RT_MANIFEST -// bpo-45220: Cannot reliably #include RT_MANIFEST from -// anywhere, so we hardcode it -#define RT_MANIFEST 24 -#endif - -// Include the manifest file that indicates we support all -// current versions of Windows. -1 RT_MANIFEST "python.manifest" - -///////////////////////////////////////////////////////////////////////////// -// -// Version -// - -VS_VERSION_INFO VERSIONINFO - FILEVERSION PYVERSION64 - PRODUCTVERSION PYVERSION64 - FILEFLAGSMASK 0x3fL -#ifdef _DEBUG - FILEFLAGS VS_FF_DEBUG -#else - FILEFLAGS 0x0L -#endif - FILEOS VOS__WINDOWS32 - FILETYPE VFT_APP - FILESUBTYPE 0x0L -BEGIN - BLOCK "StringFileInfo" - BEGIN - BLOCK "000004b0" - BEGIN - VALUE "CompanyName", PYTHON_COMPANY "\0" - VALUE "FileDescription", "Python\0" - VALUE "FileVersion", PYTHON_VERSION - VALUE "InternalName", "Python Launcher Shell Extension\0" - VALUE "LegalCopyright", PYTHON_COPYRIGHT "\0" - VALUE "OriginalFilename", "pyshellext" PYTHON_DEBUG_EXT ".dll\0" - VALUE "ProductName", "Python\0" - VALUE "ProductVersion", PYTHON_VERSION - END - END - BLOCK "VarFileInfo" - BEGIN - VALUE "Translation", 0x0, 1200 - END -END \ No newline at end of file diff --git a/PC/store_info.txt b/PC/store_info.txt deleted file mode 100644 index d150ba17cbe62dd..000000000000000 --- a/PC/store_info.txt +++ /dev/null @@ -1,156 +0,0 @@ -# Overview - -NOTE: This file requires more content. - -Since Python 3.7.2, releases have been made through the Microsoft Store -to allow easy installation on Windows 10.0.17763.0 and later. - -# Building - -To build the store package, the PC/layout script should be used. -Execute the directory with the build of Python to package, and pass -"-h" for full command-line options. - -To sideload test builds, you will need a local certificate. -Instructions are available at -https://docs.microsoft.com/windows/uwp/packaging/create-certificate-package-signing. - -After exporting your certificate, you will need the subject name and -SHA256 hash. The `certutil -dump ` command will display this -information. - -To build for sideloading, use these commands in PowerShell: - -``` -$env:APPX_DATA_PUBLISHER= -$env:APPX_DATA_SHA256= -$env:SigningCertificateFile= - -python PC/layout --copy --include-appxmanifest -Tools/msi/make_appx.ps1 python.msix -sign - -Add-AppxPackage python.msix -``` - -(Note that only the last command requires PowerShell, and the others -can be used from Command Prompt. You can also double-click to install -the final package.) - -To build for publishing to the Store, use these commands: - -``` -$env:APPX_DATA_PUBLISHER = $null -$env:APPX_DATA_SHA256 = $null - -python PC/layout --copy --preset-appxmanifest --precompile -Tools/msi/make_appx.ps1 python.msix -``` - -Note that this package cannot be installed locally. It may only be -added to a submission for the store. - - -# Submission Metadata - -This file contains the text that we use to fill out the store listing -for the Microsoft Store. It needs to be entered manually when creating -a new submission via the dashboard at -https://partner.microsoft.com/dashboard. - -We keep it here for convenience and to allow it to be updated via pull -requests. - -When submitting a new app, the HeadlessAppBypass waiver will be needed. -To request this, send an email to PartnerOps@microsoft.com with the app -ID (12 character token available from the dashboard). The waiver needs -to be applied *before* uploading the package (as of November 2019). - -Ensure that the new app is named "Python.3.X", where X is the minor -version of the release. If the name provided initially does not match -the name used when building the package, the upload will fail. The -display name shown to users can be set later. - -## Title - -Python 3.9 - -## Short Title - -Python - -## Description - -Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python’s elegant syntax and dynamic typing, together with its interpreted nature, make it an ideal language for scripting and rapid application development in many areas on most platforms. - -The Python interpreter and the extensive standard library are freely available in source or binary form for all major platforms from the Python web site, https://www.python.org/, and may be freely distributed. The same site also contains distributions of and pointers to many free third party Python modules, programs and tools, and additional documentation. - -The Python interpreter is easily extended with new functions and data types implemented in C or C++ (or other languages callable from C). Python is also suitable as an extension language for customizable applications. - -## ShortDescription - -The Python 3.9 interpreter and runtime. - -## Copyright Trademark Information - -(c) Python Software Foundation - -## Additional License Terms - -Visit https://docs.python.org/3.9/license.html for latest license terms. - -PSF LICENSE AGREEMENT FOR PYTHON 3.9 - -1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and - the Individual or Organization ("Licensee") accessing and otherwise using Python - 3.9 software in source or binary form and its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF hereby - grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, - analyze, test, perform and/or display publicly, prepare derivative works, - distribute, and otherwise use Python 3.9 alone or in any derivative - version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001 Python Software Foundation; All Rights - Reserved" are retained in Python 3.9 alone or in any derivative version - prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on or - incorporates Python 3.9 or any part thereof, and wants to make the - derivative work available to others as provided herein, then Licensee hereby - agrees to include in any such work a brief summary of the changes made to Python - 3.9. - -4. PSF is making Python 3.9 available to Licensee on an "AS IS" basis. - PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF - EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR - WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE - USE OF PYTHON 3.9 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 3.9 - FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF - MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 3.9, OR ANY DERIVATIVE - THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material breach of - its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any relationship - of agency, partnership, or joint venture between PSF and Licensee. This License - Agreement does not grant permission to use PSF trademarks or trade name in a - trademark sense to endorse or promote products or services of Licensee, or any - third party. - -8. By copying, installing or otherwise using Python 3.9, Licensee agrees - to be bound by the terms and conditions of this License Agreement. - -## Features - -* Easy to install Python runtime -* Supported by core CPython team -* Find Python, Pip and Idle on PATH - -## Search Terms - -* Python -* Scripting -* Interpreter - diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 04bd36404366a57..246a07785168161 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -284,6 +284,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 7710acf43b5137c..35082f57f54e9d8 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -496,6 +496,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index f3e423fa04668ec..cd58c3523e8c160 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -101,6 +101,7 @@ + diff --git a/PCbuild/_testinternalcapi.vcxproj.filters b/PCbuild/_testinternalcapi.vcxproj.filters index 7ab242c2c230b67..4082c36234bdf0f 100644 --- a/PCbuild/_testinternalcapi.vcxproj.filters +++ b/PCbuild/_testinternalcapi.vcxproj.filters @@ -30,6 +30,9 @@ Source Files + + Source Files + diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 7ddcee6d9735ce6..785bb151e081293 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -97,6 +97,7 @@ + diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 66a0a47d8e5548b..51dc9950a103769 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -12,6 +12,7 @@ + diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 47399fe65d1e542..6bedd21299a73e9 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -57,8 +57,8 @@ if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.7 set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.53.2.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-9.0.3.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-9.0.3.1 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-9.0.4.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-9.0.4.1 set libraries=%libraries% xz-5.8.1.1 set libraries=%libraries% zlib-ng-2.2.4 set libraries=%libraries% zstd-1.5.7 @@ -80,7 +80,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.7 -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-9.0.3.0 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-9.0.4.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-21.1.4.0 diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 9d077bbd3f0ba27..53aec0276beaee2 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -63,10 +63,6 @@ - - - - diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 09a989d38648df4..e2cac3158831eed 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -111,16 +111,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xxlimited", "xxlimited.vcxp EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testbuffer", "_testbuffer.vcxproj", "{A2697BD3-28C1-4AEC-9106-8B748639FD16}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pylauncher", "pylauncher.vcxproj", "{7B2727B5-5A3F-40EE-A866-43A13CD31446}" - ProjectSection(ProjectDependencies) = postProject - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} = {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pywlauncher", "pywlauncher.vcxproj", "{1D4B18D3-7C12-4ECB-9179-8531FF876CE6}" - ProjectSection(ProjectDependencies) = postProject - {7B2727B5-5A3F-40EE-A866-43A13CD31446} = {7B2727B5-5A3F-40EE-A866-43A13CD31446} - EndProjectSection -EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_module", "_freeze_module.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}" @@ -131,8 +121,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testmultiphase", "_testmul EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testsinglephase", "_testsinglephase.vcxproj", "{2097F1C1-597C-4167-93E3-656A7D6339B2}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyshellext", "pyshellext.vcxproj", "{0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}" -EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testconsole", "_testconsole.vcxproj", "{B244E787-C445-441C-BDF4-5A4F1A3A1E51}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" @@ -1079,70 +1067,6 @@ Global {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|Win32.Build.0 = Release|Win32 {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|x64.ActiveCfg = Release|x64 {A2697BD3-28C1-4AEC-9106-8B748639FD16}.Release|x64.Build.0 = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|ARM.ActiveCfg = Debug|ARM - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|ARM.Build.0 = Debug|ARM - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|ARM64.ActiveCfg = Debug|ARM64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|ARM64.Build.0 = Debug|ARM64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|Win32.ActiveCfg = Debug|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|Win32.Build.0 = Debug|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|x64.ActiveCfg = Debug|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Debug|x64.Build.0 = Debug|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|ARM.Build.0 = PGInstrument|ARM - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|Win32.Build.0 = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|x64.ActiveCfg = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGInstrument|x64.Build.0 = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|ARM.Build.0 = PGUpdate|ARM - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|Win32.Build.0 = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|x64.ActiveCfg = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.PGUpdate|x64.Build.0 = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|ARM.ActiveCfg = Release|ARM - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|ARM.Build.0 = Release|ARM - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|ARM64.ActiveCfg = Release|ARM64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|ARM64.Build.0 = Release|ARM64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|Win32.ActiveCfg = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|Win32.Build.0 = Release|Win32 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|x64.ActiveCfg = Release|x64 - {7B2727B5-5A3F-40EE-A866-43A13CD31446}.Release|x64.Build.0 = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|ARM.ActiveCfg = Debug|ARM - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|ARM.Build.0 = Debug|ARM - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|ARM64.ActiveCfg = Debug|ARM64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|ARM64.Build.0 = Debug|ARM64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|Win32.ActiveCfg = Debug|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|Win32.Build.0 = Debug|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|x64.ActiveCfg = Debug|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Debug|x64.Build.0 = Debug|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|ARM.Build.0 = PGInstrument|ARM - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|Win32.ActiveCfg = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|Win32.Build.0 = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|x64.ActiveCfg = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGInstrument|x64.Build.0 = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|ARM.Build.0 = PGUpdate|ARM - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|Win32.ActiveCfg = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|Win32.Build.0 = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|x64.ActiveCfg = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.PGUpdate|x64.Build.0 = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|ARM.ActiveCfg = Release|ARM - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|ARM.Build.0 = Release|ARM - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|ARM64.ActiveCfg = Release|ARM64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|ARM64.Build.0 = Release|ARM64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.ActiveCfg = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|Win32.Build.0 = Release|Win32 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.ActiveCfg = Release|x64 - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6}.Release|x64.Build.0 = Release|x64 {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM.ActiveCfg = Debug|ARM {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM.Build.0 = Debug|ARM {19C0C13F-47CA-4432-AFF3-799A296A4DDC}.Debug|ARM64.ActiveCfg = Debug|ARM64 @@ -1293,38 +1217,6 @@ Global {2097F1C1-597C-4167-93E3-656A7D6339B2}.Release|Win32.Build.0 = Release|Win32 {2097F1C1-597C-4167-93E3-656A7D6339B2}.Release|x64.ActiveCfg = Release|x64 {2097F1C1-597C-4167-93E3-656A7D6339B2}.Release|x64.Build.0 = Release|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|ARM.ActiveCfg = Debug|ARM - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|ARM.Build.0 = Debug|ARM - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|ARM64.ActiveCfg = Debug|ARM64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|ARM64.Build.0 = Debug|ARM64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.ActiveCfg = Debug|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|Win32.Build.0 = Debug|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|x64.ActiveCfg = Debug|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Debug|x64.Build.0 = Debug|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|ARM.ActiveCfg = PGInstrument|ARM - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|ARM.Build.0 = PGInstrument|ARM - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|ARM64.ActiveCfg = PGInstrument|ARM64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|ARM64.Build.0 = PGInstrument|ARM64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|Win32.ActiveCfg = PGInstrument|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|Win32.Build.0 = PGInstrument|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|x64.ActiveCfg = PGInstrument|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGInstrument|x64.Build.0 = PGInstrument|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|ARM.ActiveCfg = PGUpdate|ARM - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|ARM.Build.0 = PGUpdate|ARM - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|ARM64.ActiveCfg = PGUpdate|ARM64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|ARM64.Build.0 = PGUpdate|ARM64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|Win32.ActiveCfg = PGUpdate|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|Win32.Build.0 = PGUpdate|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|x64.ActiveCfg = PGUpdate|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.PGUpdate|x64.Build.0 = PGUpdate|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|ARM.ActiveCfg = Release|ARM - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|ARM.Build.0 = Release|ARM - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|ARM64.ActiveCfg = Release|ARM64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|ARM64.Build.0 = Release|ARM64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|Win32.ActiveCfg = Release|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|Win32.Build.0 = Release|Win32 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.ActiveCfg = Release|x64 - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782}.Release|x64.Build.0 = Release|x64 {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|ARM.ActiveCfg = Debug|ARM {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|ARM.Build.0 = Debug|ARM {B244E787-C445-441C-BDF4-5A4F1A3A1E51}.Debug|ARM64.ActiveCfg = Debug|ARM64 diff --git a/PCbuild/pylauncher.vcxproj b/PCbuild/pylauncher.vcxproj deleted file mode 100644 index ca422ef4d02e932..000000000000000 --- a/PCbuild/pylauncher.vcxproj +++ /dev/null @@ -1,114 +0,0 @@ - - - - - Debug - ARM - - - Debug - ARM64 - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - ARM - - - PGInstrument - ARM64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - ARM - - - PGUpdate - ARM64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - ARM - - - Release - ARM64 - - - Release - Win32 - - - Release - x64 - - - - {7B2727B5-5A3F-40EE-A866-43A13CD31446} - pylauncher - py - false - - - - - Application - Unicode - - - - - - ClCompile - - - - - - - - - _CONSOLE;%(PreprocessorDefinitions) - MultiThreaded - - - shell32.lib;pathcch.lib;%(AdditionalDependencies) - Console - - - - - - - - - - - - - - - diff --git a/PCbuild/pylauncher.vcxproj.filters b/PCbuild/pylauncher.vcxproj.filters deleted file mode 100644 index f10f6d5669e283c..000000000000000 --- a/PCbuild/pylauncher.vcxproj.filters +++ /dev/null @@ -1,28 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - - - Resource Files - - - - - Resource Files - - - \ No newline at end of file diff --git a/PCbuild/pyshellext.vcxproj b/PCbuild/pyshellext.vcxproj deleted file mode 100644 index ea432d6bc9a3f33..000000000000000 --- a/PCbuild/pyshellext.vcxproj +++ /dev/null @@ -1,117 +0,0 @@ - - - - - Debug - ARM - - - Debug - ARM64 - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - ARM - - - PGInstrument - ARM64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - ARM - - - PGUpdate - ARM64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - ARM - - - Release - ARM64 - - - Release - Win32 - - - Release - x64 - - - - {0F6EE4A4-C75F-4578-B4B3-2D64F4B9B782} - pyshellext - pyshellext - false - - - - - DynamicLibrary - Unicode - - - - - - ClCompile - - - - - - - - - _CONSOLE;%(PreprocessorDefinitions) - - - version.lib;shlwapi.lib;%(AdditionalDependencies) - Console - ..\PC\pyshellext.def - - - true - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/PCbuild/pyshellext.vcxproj.filters b/PCbuild/pyshellext.vcxproj.filters deleted file mode 100644 index 77cd3060857a80b..000000000000000 --- a/PCbuild/pyshellext.vcxproj.filters +++ /dev/null @@ -1,28 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - - - Resource Files - - - - - Source Files - - - \ No newline at end of file diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 47296a93424a8f6..7fec674c550e027 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -332,6 +332,7 @@ + @@ -703,6 +704,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 124f6c7bb90de6c..cb0f6fce86df51a 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -897,6 +897,9 @@ Include\internal + + Include\internal + Include\internal @@ -1622,6 +1625,9 @@ Python + + Python + Python diff --git a/PCbuild/pywlauncher.vcxproj b/PCbuild/pywlauncher.vcxproj deleted file mode 100644 index 1694548935a1653..000000000000000 --- a/PCbuild/pywlauncher.vcxproj +++ /dev/null @@ -1,114 +0,0 @@ - - - - - Debug - ARM - - - Debug - ARM64 - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - ARM - - - PGInstrument - ARM64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - ARM - - - PGUpdate - ARM64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - ARM - - - Release - ARM64 - - - Release - Win32 - - - Release - x64 - - - - {1D4B18D3-7C12-4ECB-9179-8531FF876CE6} - pywlauncher - pyw - false - - - - - Application - Unicode - - - - - - ClCompile - - - - - - - - - _WINDOWS;%(PreprocessorDefinitions) - MultiThreaded - - - shell32.lib;pathcch.lib;%(AdditionalDependencies) - Windows - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/PCbuild/pywlauncher.vcxproj.filters b/PCbuild/pywlauncher.vcxproj.filters deleted file mode 100644 index f10f6d5669e283c..000000000000000 --- a/PCbuild/pywlauncher.vcxproj.filters +++ /dev/null @@ -1,28 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} - rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - - - - - Source Files - - - - - Resource Files - - - - - Resource Files - - - \ No newline at end of file diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 7c5eab2eb75a1de..7751bdeb1a3077b 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -23,7 +23,7 @@ external dependencies. To build, simply run the "build.bat" script without any arguments. After this succeeds, you can open the "pcbuild.sln" solution in Visual Studio to continue development. -To build an installer package, refer to the README in the Tools/msi folder. +To build an installer package, refer to PC/layout. The solution currently supports two platforms. The Win32 platform is used to build standard x86-compatible 32-bit binaries, output into the @@ -140,12 +140,6 @@ CPython in different ways: pythonw pythonw.exe, a variant of python.exe that doesn't open a Command Prompt window -pylauncher - py.exe, the Python Launcher for Windows, see - https://docs.python.org/3/using/windows.html#launcher -pywlauncher - pyw.exe, a variant of py.exe that doesn't open a Command Prompt - window _testembed _testembed.exe, a small program that embeds Python for testing purposes, used by test_capi.py @@ -156,8 +150,6 @@ _freeze_module _freeze_module.exe, used to regenerate frozen modules in Python after changes have been made to the corresponding source files (e.g. Lib\importlib\_bootstrap.py). -pyshellext - pyshellext.dll, the shell extension deployed with the launcher python3dll python3.dll, the PEP 384 Stable ABI dll (not installed on free-threaded builds) @@ -247,7 +239,7 @@ _sqlite3 https://www.sqlite.org/ _tkinter - Wraps version 9.0.3 of the Tk windowing system, which is downloaded + Wraps version 9.0.4 of the Tk windowing system, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. @@ -420,8 +412,6 @@ _testclinic_limited extension, the file Modules/_testclinic_limited.c: * Save and exit Visual Studio. * Add `;_testclinic_limited` to `` in PCbuild\pcbuild.proj. -* Update "exts" in Tools\msi\lib\lib_files.wxs file or in - Tools\msi\test\test_files.wxs file (for tests). * PC\layout\main.py needs updating if you add a test-only extension whose name doesn't start with "_test". * Add the extension to PCbuild\readme.txt (this file). diff --git a/PCbuild/tcltk.props b/PCbuild/tcltk.props index 28e8c0db4d1eafd..1d6b4f57f09f747 100644 --- a/PCbuild/tcltk.props +++ b/PCbuild/tcltk.props @@ -2,7 +2,7 @@ - 9.0.3.0 + 9.0.4.0 $(TclVersion) $([System.Version]::Parse($(TclVersion)).Major) $([System.Version]::Parse($(TclVersion)).Minor) diff --git a/PCbuild/venvlauncher.vcxproj b/PCbuild/venvlauncher.vcxproj index a2e8ffa82b10eb7..7afe923406b1d18 100644 --- a/PCbuild/venvlauncher.vcxproj +++ b/PCbuild/venvlauncher.vcxproj @@ -110,10 +110,7 @@ - - - - + diff --git a/PCbuild/venvlauncher.vcxproj.filters b/PCbuild/venvlauncher.vcxproj.filters index bc98687a6da0d3e..62839fd6b8e863f 100644 --- a/PCbuild/venvlauncher.vcxproj.filters +++ b/PCbuild/venvlauncher.vcxproj.filters @@ -9,12 +9,7 @@ - - Resource Files - - - - + Resource Files diff --git a/PCbuild/venvwlauncher.vcxproj b/PCbuild/venvwlauncher.vcxproj index f2aaf83fe2b3785..3aa0bb2cdefeaa8 100644 --- a/PCbuild/venvwlauncher.vcxproj +++ b/PCbuild/venvwlauncher.vcxproj @@ -110,10 +110,7 @@ - - - - + diff --git a/PCbuild/venvwlauncher.vcxproj.filters b/PCbuild/venvwlauncher.vcxproj.filters index 5193c38b12d53d5..babded59633f1f1 100644 --- a/PCbuild/venvwlauncher.vcxproj.filters +++ b/PCbuild/venvwlauncher.vcxproj.filters @@ -14,12 +14,7 @@ - - Resource Files - - - - + Resource Files diff --git a/Parser/parser.c b/Parser/parser.c index 4f4fd3ed46d1f02..800db5b490fea98 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -9355,7 +9355,7 @@ real_number_rule(Parser *p) return _res; } -// imaginary_number: NUMBER | '+' NUMBER +// imaginary_number: NUMBER static expr_ty imaginary_number_rule(Parser *p) { @@ -9392,33 +9392,6 @@ imaginary_number_rule(Parser *p) D(fprintf(stderr, "%*c%s imaginary_number[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NUMBER")); } - { // '+' NUMBER - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> imaginary_number[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+' NUMBER")); - Token * _literal; - expr_ty imag; - if ( - (_literal = _PyPegen_expect_token(p, 14)) // token='+' - && - (imag = _PyPegen_number_token(p)) // NUMBER - ) - { - D(fprintf(stderr, "%*c+ imaginary_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+' NUMBER")); - _res = _PyPegen_ensure_imaginary ( p , imag ); - if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { - p->error_indicator = 1; - p->level--; - return NULL; - } - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s imaginary_number[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+' NUMBER")); - } _res = NULL; done: p->level--; diff --git a/Parser/pegen.c b/Parser/pegen.c index 165dcb9f9950955..fcec810037e98d4 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -12,6 +12,16 @@ #include "tokenizer/helpers.h" #include "pegen.h" +#define IDENTIFIER_CACHE_SIZE 2048 // Must be a power of two. +#define IDENTIFIER_CACHE_MAX_PROBES 8 + +struct _identifier_cache_entry { + const char *key; // Borrowed from arena-owned token bytes. + Py_ssize_t len; + Py_hash_t hash; + PyObject *value; // Borrowed from an arena-owned identifier. +}; + // Internal parser functions asdl_stmt_seq* @@ -90,6 +100,7 @@ _PyPegen_insert_memo(Parser *p, int mark, int type, void *node) m->mark = p->mark; m->next = p->tokens[mark]->memo; p->tokens[mark]->memo = m; + p->tokens[mark]->memo_mask |= 1ULL << (type & 63); return 0; } @@ -350,6 +361,10 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres) Token *t = p->tokens[p->mark]; + if (!(t->memo_mask & (1ULL << (type & 63)))) { + return 0; + } + for (Memo *m = t->memo; m != NULL; m = m->next) { if (m->type == type) { #if defined(Py_DEBUG) @@ -572,11 +587,44 @@ _PyPegen_name_from_token(Parser *p, Token* t) p->error_indicator = 1; return NULL; } + // Identifiers repeat constantly; a small span-keyed cache skips the + // UTF-8 decode + intern for repeated occurrences. Keys point into + // arena-owned token bytes and values are arena-owned interned strings, + // so borrowed references are valid for the lifetime of the parse + // (including the second error pass, which reuses parser and arena). + Py_ssize_t len = PyBytes_GET_SIZE(t->bytes); + Py_hash_t hash = PyObject_Hash(t->bytes); + if (hash == -1) { + p->error_indicator = 1; + return NULL; + } + IdentifierCacheEntry *free_slot = NULL; + size_t idx = (size_t)hash & (IDENTIFIER_CACHE_SIZE - 1); + for (int probe = 0; probe < IDENTIFIER_CACHE_MAX_PROBES; probe++) { + IdentifierCacheEntry *entry = &p->identifier_cache[ + (idx + probe) & (IDENTIFIER_CACHE_SIZE - 1)]; + if (entry->key == NULL) { + free_slot = entry; + break; + } + if (entry->hash == hash && entry->len == len && + memcmp(entry->key, s, len) == 0) + { + return _PyAST_Name(entry->value, Load, t->lineno, t->col_offset, + t->end_lineno, t->end_col_offset, p->arena); + } + } PyObject *id = _PyPegen_new_identifier(p, s); if (id == NULL) { p->error_indicator = 1; return NULL; } + if (free_slot != NULL) { + free_slot->key = s; + free_slot->len = len; + free_slot->hash = hash; + free_slot->value = id; + } return _PyAST_Name(id, Load, t->lineno, t->col_offset, t->end_lineno, t->end_col_offset, p->arena); } @@ -844,6 +892,15 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, p->flags = flags; p->feature_version = feature_version; p->known_err_token = NULL; + p->identifier_cache = PyMem_Calloc( + IDENTIFIER_CACHE_SIZE, sizeof(*p->identifier_cache)); + if (p->identifier_cache == NULL) { + growable_comment_array_deallocate(&p->type_ignore_comments); + PyMem_Free(p->tokens[0]); + PyMem_Free(p->tokens); + PyMem_Free(p); + return (Parser *) PyErr_NoMemory(); + } p->level = 0; p->call_invalid_rules = 0; p->last_stmt_location.lineno = 0; @@ -859,6 +916,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags, void _PyPegen_Parser_Free(Parser *p) { + PyMem_Free(p->identifier_cache); Py_XDECREF(p->normalize); for (int i = 0; i < p->size; i++) { PyMem_Free(p->tokens[i]); diff --git a/Parser/pegen.h b/Parser/pegen.h index cbf44ba474fa395..3cca698692bf6bf 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -42,6 +42,10 @@ typedef struct { int level; int lineno, col_offset, end_lineno, end_col_offset; Memo *memo; + // Filter over the rule types present in `memo` (bit `type & 63` is set + // for every entry): lets lookups skip walking the list on definite + // misses, which are the common case. + uint64_t memo_mask; PyObject *metadata; } Token; @@ -67,6 +71,8 @@ typedef struct { int end_col_offset; } location; +typedef struct _identifier_cache_entry IdentifierCacheEntry; + typedef struct { struct tok_state *tok; Token **tokens; @@ -91,6 +97,7 @@ typedef struct { int call_invalid_rules; int debug; location last_stmt_location; + IdentifierCacheEntry *identifier_cache; } Parser; typedef struct { diff --git a/Platforms/emscripten/config.site-wasm32-emscripten b/Platforms/emscripten/config.site-wasm32-emscripten index f69dbb8e779a426..3c9a1c2c2568b0e 100644 --- a/Platforms/emscripten/config.site-wasm32-emscripten +++ b/Platforms/emscripten/config.site-wasm32-emscripten @@ -21,9 +21,6 @@ ac_cv_func_eventfd=no ac_cv_func_memfd_create=no ac_cv_func_prlimit=no -# unsupported syscall, https://github.com/emscripten-core/emscripten/issues/13393 -ac_cv_func_shutdown=no - # The rest is based on pyodide # https://github.com/pyodide/pyodide/blob/main/cpython/pyconfig.undefs.h diff --git a/Platforms/emscripten/config.toml b/Platforms/emscripten/config.toml index 0c65a6623c9798a..a6232c3bf594098 100644 --- a/Platforms/emscripten/config.toml +++ b/Platforms/emscripten/config.toml @@ -1,7 +1,7 @@ # Any data that can vary between Python versions is to be kept in this file. # This allows for blanket copying of the Emscripten build code between supported # Python versions. -emscripten-version = "6.0.2" +emscripten-version = "6.0.4" node-version = "24" test-args = [ "-m", "test", diff --git a/Python/dynload_shlib.c b/Python/dynload_shlib.c index 0ff88ad330fd09c..f7f7c9229200857 100644 --- a/Python/dynload_shlib.c +++ b/Python/dynload_shlib.c @@ -46,8 +46,14 @@ const char *_PyImport_DynLoadFiletab[] = { "." ALT_SOABI ".so", #endif #ifndef Py_GIL_DISABLED +#ifdef SOABI_PLATFORM + ".abi" PYTHON_ABI_STRING "-" SOABI_PLATFORM ".so", +#endif /* SOABI_PLATFORM */ ".abi" PYTHON_ABI_STRING ".so", #endif /* Py_GIL_DISABLED */ +#ifdef SOABI_PLATFORM + ".abi" PYTHON_ABI_STRING "t-" SOABI_PLATFORM ".so", +#endif /* SOABI_PLATFORM */ ".abi" PYTHON_ABI_STRING "t.so", ".so", #endif /* __CYGWIN__ */ diff --git a/Python/fileutils.c b/Python/fileutils.c index 98ed66eff94ee10..5cbfd8e6ce4fa08 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1103,50 +1103,41 @@ typedef union { void -_Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, +_Py_attribute_data_to_stat(FILE_STANDARD_INFO* standard_info, ULONG reparse_tag, FILE_BASIC_INFO *basic_info, FILE_ID_INFO *id_info, struct _Py_stat_struct *result) { memset(result, 0, sizeof(*result)); - result->st_mode = attributes_to_mode(info->dwFileAttributes); - result->st_size = (((__int64)info->nFileSizeHigh)<<32) + info->nFileSizeLow; - result->st_dev = id_info ? id_info->VolumeSerialNumber : info->dwVolumeSerialNumber; - result->st_rdev = 0; + + result->st_size = standard_info->EndOfFile.QuadPart; + result->st_nlink = standard_info->NumberOfLinks; + /* st_ctime is deprecated, but we preserve the legacy value in our caller, not here */ - if (basic_info) { - LARGE_INTEGER_to_time_t_nsec(&basic_info->CreationTime, &result->st_birthtime, &result->st_birthtime_nsec); - LARGE_INTEGER_to_time_t_nsec(&basic_info->ChangeTime, &result->st_ctime, &result->st_ctime_nsec); - LARGE_INTEGER_to_time_t_nsec(&basic_info->LastWriteTime, &result->st_mtime, &result->st_mtime_nsec); - LARGE_INTEGER_to_time_t_nsec(&basic_info->LastAccessTime, &result->st_atime, &result->st_atime_nsec); - } else { - FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_birthtime, &result->st_birthtime_nsec); - FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); - FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); - } - result->st_nlink = info->nNumberOfLinks; + LARGE_INTEGER_to_time_t_nsec(&basic_info->CreationTime, &result->st_birthtime, &result->st_birthtime_nsec); + LARGE_INTEGER_to_time_t_nsec(&basic_info->ChangeTime, &result->st_ctime, &result->st_ctime_nsec); + LARGE_INTEGER_to_time_t_nsec(&basic_info->LastWriteTime, &result->st_mtime, &result->st_mtime_nsec); + LARGE_INTEGER_to_time_t_nsec(&basic_info->LastAccessTime, &result->st_atime, &result->st_atime_nsec); if (id_info) { + result->st_dev = id_info->VolumeSerialNumber; id_128_to_ino file_id; file_id.id = id_info->FileId; result->st_ino = file_id.st_ino; result->st_ino_high = file_id.st_ino_high; } - if (!result->st_ino && !result->st_ino_high) { - /* should only occur for DirEntry_from_find_data, in which case the - index is likely to be zero anyway. */ - result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow; - } + + result->st_file_attributes = basic_info->FileAttributes; + result->st_mode = attributes_to_mode(result->st_file_attributes); /* bpo-37834: Only actual symlinks set the S_IFLNK flag. But lstat() will open other name surrogate reparse points without traversing them. To detect/handle these, check st_file_attributes and st_reparse_tag. */ result->st_reparse_tag = reparse_tag; - if (info->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT && + if (result->st_file_attributes & FILE_ATTRIBUTE_REPARSE_POINT && reparse_tag == IO_REPARSE_TAG_SYMLINK) { /* set the bits that make this a symlink */ result->st_mode = (result->st_mode & ~S_IFMT) | S_IFLNK; } - result->st_file_attributes = info->dwFileAttributes; } void @@ -1231,9 +1222,9 @@ int _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) { #ifdef MS_WINDOWS - BY_HANDLE_FILE_INFORMATION info; - FILE_BASIC_INFO basicInfo; - FILE_ID_INFO idInfo; + FILE_STANDARD_INFO standardInfo = {0}; + FILE_BASIC_INFO basicInfo = {0}; + FILE_ID_INFO idInfo = {0}; FILE_ID_INFO *pIdInfo = &idInfo; HANDLE h; int type; @@ -1266,7 +1257,7 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) return 0; } - if (!GetFileInformationByHandle(h, &info) || + if (!GetFileInformationByHandleEx(h,FileStandardInfo, &standardInfo, sizeof(standardInfo)) || !GetFileInformationByHandleEx(h, FileBasicInfo, &basicInfo, sizeof(basicInfo))) { /* The Win32 error is already set, but we also set errno for callers who expect it */ @@ -1279,7 +1270,7 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) pIdInfo = NULL; } - _Py_attribute_data_to_stat(&info, 0, &basicInfo, pIdInfo, status); + _Py_attribute_data_to_stat(&standardInfo, 0, &basicInfo, pIdInfo, status); return 0; #else return fstat(fd, status); diff --git a/Python/import.c b/Python/import.c index 63e23e21beb1266..5ca78a971fa54c6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4103,7 +4103,9 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) } static PyObject * -import_find_and_load(PyThreadState *tstate, PyObject *abs_name) +import_find_and_load_with_name(PyThreadState *tstate, PyObject *abs_name, + PyObject *find_and_load, + PyObject *not_found) { PyObject *mod = NULL; PyInterpreterState *interp = tstate->interp; @@ -4130,12 +4132,14 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) if (PyDTrace_IMPORT_FIND_LOAD_START_ENABLED()) PyDTrace_IMPORT_FIND_LOAD_START(PyUnicode_AsUTF8(abs_name)); - mod = PyObject_CallMethodObjArgs(IMPORTLIB(interp), &_Py_ID(_find_and_load), + mod = PyObject_CallMethodObjArgs(IMPORTLIB(interp), find_and_load, abs_name, IMPORT_FUNC(interp), NULL); - if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) + if (PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED()) { + int found = mod != NULL && mod != not_found; PyDTrace_IMPORT_FIND_LOAD_DONE(PyUnicode_AsUTF8(abs_name), - mod != NULL); + found); + } if (import_time) { PyTime_t t2; @@ -4156,6 +4160,13 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef accumulated } +static PyObject * +import_find_and_load(PyThreadState *tstate, PyObject *abs_name) +{ + return import_find_and_load_with_name( + tstate, abs_name, &_Py_ID(_find_and_load), NULL); +} + static PyObject * get_abs_name(PyThreadState *tstate, PyObject *name, PyObject *globals, int level) @@ -4439,52 +4450,70 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, return res; } -PyObject * -_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name) +_PyLazySubmoduleImportResult +_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name, + PyObject **result) { - PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(result != NULL); + *result = NULL; + + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp); if (lazy_pending == NULL) { - return NULL; + return _Py_LAZY_SUBMODULE_NOT_FOUND; } PyObject *pending_set; int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set); - if (rc <= 0) { - return NULL; + if (rc < 0) { + return _Py_LAZY_SUBMODULE_ERROR; + } + if (rc == 0) { + return _Py_LAZY_SUBMODULE_NOT_FOUND; } int contains = PySet_Contains(pending_set, attr_name); - if (contains <= 0) { + if (contains < 0) { Py_DECREF(pending_set); - return NULL; + return _Py_LAZY_SUBMODULE_ERROR; + } + if (contains == 0) { + Py_DECREF(pending_set); + return _Py_LAZY_SUBMODULE_NOT_FOUND; } PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); if (full_name == NULL) { Py_DECREF(pending_set); - return NULL; + return _Py_LAZY_SUBMODULE_ERROR; } - PyObject *mod = PyImport_ImportModuleLevelObject( - full_name, NULL, NULL, NULL, 0); + PyObject *mod = import_find_and_load_with_name( + tstate, full_name, &_Py_ID(_find_and_load_lazy_submodule), Py_None); if (mod == NULL) { Py_DECREF(pending_set); Py_DECREF(full_name); - return NULL; + remove_importlib_frames(tstate); + return _Py_LAZY_SUBMODULE_ERROR; + } + if (mod == Py_None) { + Py_DECREF(mod); + Py_DECREF(pending_set); + Py_DECREF(full_name); + return _Py_LAZY_SUBMODULE_NOT_FOUND; } - Py_DECREF(mod); if (PySet_Discard(pending_set, attr_name) < 0) { + Py_DECREF(mod); Py_DECREF(pending_set); Py_DECREF(full_name); - return NULL; + return _Py_LAZY_SUBMODULE_ERROR; } Py_DECREF(pending_set); - - PyObject *submod = PyImport_GetModule(full_name); Py_DECREF(full_name); - return submod; + *result = mod; + return _Py_LAZY_SUBMODULE_LOADED; } PyObject * diff --git a/Python/optimizer.c b/Python/optimizer.c index c9f6ebdb62f07b2..e05adb344c8d06d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1797,6 +1797,11 @@ make_cold_executor(uint16_t opcode) Py_FatalError("Cannot allocate core JIT code"); } ((_PyUOpInstruction *)cold->trace)->opcode = opcode; + // Cold executors bypass _Py_ExecutorInit(). + cold->vm_data.valid = true; + cold->vm_data.pending_deletion = 0; + cold->vm_data.code = NULL; + // This is initialized to false so we can prevent the executor // from being immediately detected as cold and invalidated. cold->vm_data.cold = false; @@ -1804,7 +1809,7 @@ make_cold_executor(uint16_t opcode) cold->jit_code = NULL; cold->jit_size = 0; if (_PyJIT_Compile(cold, cold->trace, 1)) { - Py_DECREF(cold); + _PyExecutor_Free(cold); Py_FatalError("Cannot allocate core JIT code"); } #endif diff --git a/Python/pystate.c b/Python/pystate.c index e90642fa882db72..d10b38def32911d 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -12,7 +12,7 @@ #include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interpframe.h" // _PyThreadState_HasStackSpace() -#include "pycore_object.h" // _PyType_InitCache(), _Py_ClearImmortal() +#include "pycore_object.h" // _Py_ClearImmortal() #include "pycore_obmalloc.h" // _PyMem_obmalloc_state_on_heap() #include "pycore_opcode_utils.h" // NUM_COMMON_CONSTANTS #include "pycore_optimizer.h" // JIT_CLEANUP_THRESHOLD @@ -420,8 +420,6 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime) } #endif - _PyTypes_AfterFork(); - _PyThread_AfterFork(&runtime->threads); return _PyStatus_OK(); @@ -573,7 +571,6 @@ init_interpreter(PyInterpreterState *interp, _PyEval_InitState(interp); _PyGC_InitState(&interp->gc); PyConfig_InitPythonConfig(&interp->config); - _PyType_InitCache(interp); #ifdef Py_GIL_DISABLED _Py_brc_init_state(interp); #endif diff --git a/Python/pystats.c b/Python/pystats.c index 2fac2db1b738c7e..6862521f659f202 100644 --- a/Python/pystats.c +++ b/Python/pystats.c @@ -230,9 +230,11 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Object materialize dict (str subclass): %" PRIu64 "\n", stats->dict_materialized_str_subclass); fprintf(out, "Object method cache hits: %" PRIu64 "\n", stats->type_cache_hits); fprintf(out, "Object method cache misses: %" PRIu64 "\n", stats->type_cache_misses); - fprintf(out, "Object method cache collisions: %" PRIu64 "\n", stats->type_cache_collisions); + fprintf(out, "Object method cache too big: %" PRIu64 "\n", stats->type_cache_too_big); fprintf(out, "Object method cache dunder hits: %" PRIu64 "\n", stats->type_cache_dunder_hits); fprintf(out, "Object method cache dunder misses: %" PRIu64 "\n", stats->type_cache_dunder_misses); + fprintf(out, "Object method cache invalidations: %" PRIu64 "\n", stats->type_cache_invalidations); + fprintf(out, "Object method cache resizes: %" PRIu64 "\n", stats->type_cache_resizes); } static void @@ -439,7 +441,9 @@ merge_object_stats(ObjectStats *dest, const ObjectStats *src) dest->type_cache_misses += src->type_cache_misses; dest->type_cache_dunder_hits += src->type_cache_dunder_hits; dest->type_cache_dunder_misses += src->type_cache_dunder_misses; - dest->type_cache_collisions += src->type_cache_collisions; + dest->type_cache_too_big += src->type_cache_too_big; + dest->type_cache_invalidations += src->type_cache_invalidations; + dest->type_cache_resizes += src->type_cache_resizes; dest->object_visits += src->object_visits; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 9442472b53abbe1..1e6e914b066bc5c 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -527,8 +527,8 @@ sys_addaudithook_impl(PyObject *module, PyObject *hook) /* Invoke existing audit hooks to allow them an opportunity to abort. */ if (_PySys_Audit(tstate, "sys.addaudithook", NULL) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_Exception)) { - /* We do not report errors derived from Exception */ + if (_PyErr_ExceptionMatches(tstate, PyExc_RuntimeError)) { + /* We do not report errors derived from RuntimeError */ _PyErr_Clear(tstate); Py_RETURN_NONE; } diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 9a29d14ef67678d..086dab912ecfd71 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -11,142 +11,6 @@ #include #endif -/* options */ -#ifndef _PY_USE_CV_LOCKS -#define _PY_USE_CV_LOCKS 1 /* use locks based on cond vars */ -#endif - -/* Now, define a non-recursive mutex using either condition variables - * and critical sections (fast) or using operating system mutexes - * (slow) - */ - -#if _PY_USE_CV_LOCKS - -#include "condvar.h" - -typedef struct _NRMUTEX -{ - PyMUTEX_T cs; - PyCOND_T cv; - int locked; -} NRMUTEX; -typedef NRMUTEX *PNRMUTEX; - -static PNRMUTEX -AllocNonRecursiveMutex(void) -{ - PNRMUTEX m = (PNRMUTEX)PyMem_RawMalloc(sizeof(NRMUTEX)); - if (!m) - return NULL; - if (PyCOND_INIT(&m->cv)) - goto fail; - if (PyMUTEX_INIT(&m->cs)) { - PyCOND_FINI(&m->cv); - goto fail; - } - m->locked = 0; - return m; -fail: - PyMem_RawFree(m); - return NULL; -} - -static VOID -FreeNonRecursiveMutex(PNRMUTEX mutex) -{ - if (mutex) { - PyCOND_FINI(&mutex->cv); - PyMUTEX_FINI(&mutex->cs); - PyMem_RawFree(mutex); - } -} - -static DWORD -EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) -{ - DWORD result = WAIT_OBJECT_0; - if (PyMUTEX_LOCK(&mutex->cs)) - return WAIT_FAILED; - if (milliseconds == INFINITE) { - while (mutex->locked) { - if (PyCOND_WAIT(&mutex->cv, &mutex->cs)) { - result = WAIT_FAILED; - break; - } - } - } else if (milliseconds != 0) { - /* wait at least until the deadline */ - PyTime_t timeout = (PyTime_t)milliseconds * (1000 * 1000); - PyTime_t deadline = _PyDeadline_Init(timeout); - while (mutex->locked) { - PyTime_t microseconds = _PyTime_AsMicroseconds(timeout, - _PyTime_ROUND_TIMEOUT); - if (PyCOND_TIMEDWAIT(&mutex->cv, &mutex->cs, microseconds) < 0) { - result = WAIT_FAILED; - break; - } - - timeout = _PyDeadline_Get(deadline); - if (timeout <= 0) { - break; - } - } - } - if (!mutex->locked) { - mutex->locked = 1; - result = WAIT_OBJECT_0; - } else if (result == WAIT_OBJECT_0) - result = WAIT_TIMEOUT; - /* else, it is WAIT_FAILED */ - PyMUTEX_UNLOCK(&mutex->cs); /* must ignore result here */ - return result; -} - -static BOOL -LeaveNonRecursiveMutex(PNRMUTEX mutex) -{ - BOOL result; - if (PyMUTEX_LOCK(&mutex->cs)) - return FALSE; - mutex->locked = 0; - /* condvar APIs return 0 on success. We need to return TRUE on success. */ - result = !PyCOND_SIGNAL(&mutex->cv); - PyMUTEX_UNLOCK(&mutex->cs); - return result; -} - -#else /* if ! _PY_USE_CV_LOCKS */ - -/* NR-locks based on a kernel mutex */ -#define PNRMUTEX HANDLE - -static PNRMUTEX -AllocNonRecursiveMutex(void) -{ - return CreateSemaphore(NULL, 1, 1, NULL); -} - -static VOID -FreeNonRecursiveMutex(PNRMUTEX mutex) -{ - /* No in-use check */ - CloseHandle(mutex); -} - -static DWORD -EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) -{ - return WaitForSingleObjectEx(mutex, milliseconds, FALSE); -} - -static BOOL -LeaveNonRecursiveMutex(PNRMUTEX mutex) -{ - return ReleaseSemaphore(mutex, 1, NULL); -} -#endif /* _PY_USE_CV_LOCKS */ - unsigned long PyThread_get_thread_ident(void); #ifdef PY_HAVE_THREAD_NATIVE_ID diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 8496f91db2eec2b..de93178576e6ec7 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -399,8 +399,8 @@ PyThread_get_thread_native_id(void) lwpid_t native_id; native_id = _lwp_self(); #elif defined(__DragonFly__) - lwpid_t native_id; - native_id = lwp_gettid(); + // lwp_gettid() is only unique within a process, so combine it with the pid. + unsigned long native_id = (unsigned long)getpid() << 32 | lwp_gettid(); #elif defined(__sun__) && SIZEOF_LONG >= 8 unsigned long native_id = (unsigned long)getpid() << 32 | thr_self(); #endif diff --git a/Python/typecache.c b/Python/typecache.c new file mode 100644 index 000000000000000..26079de5ce341f7 --- /dev/null +++ b/Python/typecache.c @@ -0,0 +1,246 @@ +// Per-type method cache implementation + +// The cache is used for method and attribute lookups on type objects. +// The stored names are always interned strings, and the +// stored values are borrowed references to the corresponding method or attribute object. +// For static types, the cache is stored on the per-interpreter managed_static_type_state, +// and for heap types the cache is stored in the `PyTypeObject._tp_cache` field. + +#include "Python.h" +#include "pycore_typecache.h" +#include "pycore_interp.h" // PyInterpreterState +#include "pycore_pymem.h" +#include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_pyatomic_ft_wrappers.h" +#include "pycore_typeobject.h" // _PyStaticType_GetState() +#include "pycore_stats.h" // OBJECT_STAT_INC + + +// The empty cache is statically allocated and shared across all the types, +// when a type is modified, the cache of type is set to the empty cache +// and when a cache entry is inserted to the empty cache, a new cache is +// allocated for the type and the entry is inserted to the new cache. +static struct type_cache empty_cache = { + .mask = _Py_TYPECACHE_MINSIZE - 1, + .version_tag = 0, + .available = 0, + .used = 0, +}; + +static inline uint32_t +cache_size(struct type_cache *cache) +{ + return cache->mask + 1; +} + +static inline size_t +cache_nbytes(struct type_cache *cache) +{ + return offsetof(struct type_cache, hashtable) + + (size_t)cache_size(cache) * sizeof(struct type_cache_entry); +} + +static struct type_cache * +cache_allocate(uint32_t size) +{ + // size must be a power of two + assert((size & (size - 1)) == 0); + size_t nbytes = offsetof(struct type_cache, hashtable) + + (size_t)size * sizeof(struct type_cache_entry); + struct type_cache *cache = PyMem_Calloc(1, nbytes); + if (cache == NULL) { + return NULL; + } + cache->mask = size - 1; + // load factor of 0.75 + cache->available = size - (size >> 2); + cache->used = 0; + return cache; +} + +static void +cache_free_delayed(struct type_cache *cache) +{ + if (cache == NULL || cache == &empty_cache) { + return; + } +#ifndef Py_GIL_DISABLED + // On gil-enabled builds, the cache owns strong references to the interned strings, + // so we need to decref them before freeing the cache memory. + for (uint32_t i = 0; i < cache_size(cache); i++) { + if (cache->hashtable[i].name != NULL) { + Py_DECREF(cache->hashtable[i].name); + } + } +#endif + // Delay the freeing of old cache for concurrent lock-free readers + _PyMem_FreeDelayed(cache, cache_nbytes(cache)); +} + + +static inline void ** +cache_slot(PyTypeObject *type) +{ + if (type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + managed_static_type_state *state = _PyStaticType_GetState(interp, type); + assert(state != NULL); + return &state->_tp_cache; + } + return &type->_tp_cache; +} + +static inline struct type_cache * +cache_get(PyTypeObject *type) +{ + return (struct type_cache *)FT_ATOMIC_LOAD_PTR_ACQUIRE(*cache_slot(type)); +} + +static inline void +cache_set(PyTypeObject *type, struct type_cache *cache) +{ + FT_ATOMIC_STORE_PTR_RELEASE(*cache_slot(type), cache); +} + +void +_PyTypeCache_InitType(PyTypeObject *type) +{ + *cache_slot(type) = &empty_cache; +} + +static inline void +cache_insert(struct type_cache *cache, PyObject *name, + PyObject *value) +{ + Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(name); + assert(hash != -1); + uint32_t index = hash & cache->mask; + for (;;) { + if (cache->hashtable[index].name == NULL) { +#ifndef Py_GIL_DISABLED + // On free-threading, all interned strings are immortal. + Py_INCREF(name); +#endif + cache->hashtable[index].value = value; + FT_ATOMIC_STORE_PTR_RELEASE(cache->hashtable[index].name, name); + cache->used++; + cache->available--; + return; + } + else if (cache->hashtable[index].name == name) { + /* someone else added the entry before us. */ + return; + } + index = (index + 1) & cache->mask; + } +} + +static inline int +cache_resize(PyTypeObject *type, struct type_cache *cache) +{ + uint32_t old_size = cache_size(cache); + uint32_t new_size; + if (cache->used == 0) { + // the cache is the empty cache, we need to allocate a new cache with the minimum size + new_size = _Py_TYPECACHE_MINSIZE; + } + else { + // double the cache size when resizing + new_size = old_size * 2; + } + if (new_size > _Py_TYPECACHE_MAXSIZE) { + // The new size is too big, don't resize and just return. + OBJECT_STAT_INC(type_cache_too_big); + return -1; + } + struct type_cache *new_cache = cache_allocate(new_size); + if (new_cache == NULL) { + return -1; + } + OBJECT_STAT_INC(type_cache_resizes); + for (uint32_t i = 0; i < old_size; i++) { + if (cache->hashtable[i].name != NULL) { + cache_insert(new_cache, cache->hashtable[i].name, cache->hashtable[i].value); + } + } + new_cache->version_tag = type->tp_version_tag; + cache_set(type, new_cache); + cache_free_delayed(cache); + return 0; +} + +// Insert a new entry to the type cache. +// The TYPE_LOCK should be held while calling this function. +void +_PyTypeCache_Insert(PyTypeObject *type, PyObject *name, PyObject *value) +{ + struct type_cache *cache = cache_get(type); + // If the cache is full, resize it before inserting the new entry. + // this also handles the case of empty cache where available is 0 but there are no entries. + if (cache->available == 0) { + if (cache_resize(type, cache) == -1) { + // out of memory, don't cache the value + return; + } + cache = cache_get(type); + assert(cache->available > 0); + } + assert(cache->version_tag == type->tp_version_tag); + cache_insert(cache, name, value); +} + + +// Lookup the given name in the type cache. +// The cache is lock-free so it is possible that cache becomes stale during the lookup, +// to prevent returning stale cache entry, the cache version is compared with the type version tag. +struct _PyTypeCacheLookupResult +_PyTypeCache_Lookup(PyTypeObject *type, PyObject *name) +{ + assert(PyUnicode_CheckExact(name) && PyUnicode_CHECK_INTERNED(name)); + struct _PyTypeCacheLookupResult miss = {PyStackRef_NULL, 0, 0}; + struct type_cache *cache = cache_get(type); + if (cache == NULL) { + return miss; + } + Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(name); + assert(hash != -1); + uint32_t index = hash & cache->mask; + _PyStackRef out_ref; + for (;;) { + PyObject *entry_name = FT_ATOMIC_LOAD_PTR_ACQUIRE(cache->hashtable[index].name); + if (entry_name == name) { +#ifdef Py_GIL_DISABLED + if (!_Py_TryXGetStackRef(&cache->hashtable[index].value, &out_ref)) { + return miss; + } +#else + PyObject *v = cache->hashtable[index].value; + out_ref = v ? PyStackRef_FromPyObjectNew(v) : PyStackRef_NULL; +#endif + break; + } + else if (entry_name == NULL) { + return miss; + } + index = (index + 1) & cache->mask; + } + // Check the cache version against the type version tag to maintain + // consistency with find_name_in_mro and prevent stale cache reads + if (cache->version_tag != FT_ATOMIC_LOAD_UINT_RELAXED(type->tp_version_tag)) { + PyStackRef_XCLOSE(out_ref); + return miss; + } + return (struct _PyTypeCacheLookupResult){out_ref, 1, cache->version_tag}; +} + +// Invalidate the type cache of the type. +// The cache is set to the empty cache and the old cache is freed with QSBR. +// The TYPE_LOCK should be held while calling this function. +void +_PyTypeCache_Invalidate(PyTypeObject *type) +{ + OBJECT_STAT_INC(type_cache_invalidations); + struct type_cache *cache = cache_get(type); + cache_set(type, &empty_cache); + cache_free_delayed(cache); +} diff --git a/README.rst b/README.rst index 48f86cdb86ed1d2..8505ec0a1e89dc7 100644 --- a/README.rst +++ b/README.rst @@ -76,8 +76,8 @@ to macOS framework and universal builds. Refer to `Mac/README.rst On Windows, see `PCbuild/readme.txt `_. -To build Windows installer, see `Tools/msi/README.txt -`_. +To build Windows packages, see `PC/layout/README.md +`_. If you wish, you can create a subdirectory and invoke configure from there. For example:: diff --git a/Tools/README b/Tools/README index 90acb2614820ee4..3984bce6b9b6768 100644 --- a/Tools/README +++ b/Tools/README @@ -34,10 +34,6 @@ jit Tooling for building the JIT. lockbench Benchmarks for PyMutex and critical sections. -msi Support for packaging Python as an MSI package on Windows. - -nuget Files for the NuGet package manager for .NET. - patchcheck Tools for checking and applying patches to the Python source code and verifying the integrity of patch files. diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 53d7b8fe32f89e6..4d491cc79ad0c8e 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -117,7 +117,6 @@ class Outputs: run_tests: bool = False run_ubuntu: bool = False run_wasi: bool = False - run_windows_msi: bool = False run_windows_tests: bool = False @@ -160,9 +159,6 @@ def compute_changes() -> None: if outputs.run_docs: print("Build documentation") - if outputs.run_windows_msi: - print("Build Windows MSI") - print(outputs) write_github_output(outputs) @@ -223,7 +219,6 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: run_ci_fuzz_stdlib = False run_docs = False run_windows_tests = False - run_windows_msi = False platforms_changed = set() has_platform_specific_change = True @@ -245,9 +240,6 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: run_tests = True run_windows_tests = True continue - if file.name == "reusable-windows-msi.yml": - run_windows_msi = True - continue if file.name == "reusable-macos.yml": run_tests = True platforms_changed.add("macos") @@ -288,10 +280,6 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: if doc_file: run_docs = True - # Check for changed MSI installer-related files - if file.parts[:2] == ("Tools", "msi"): - run_windows_msi = True - # Check which platform specific tests to run if run_tests: if not has_platform_specific_change or not platforms_changed: @@ -327,7 +315,6 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: run_tests=run_tests, run_ubuntu=run_ubuntu, run_wasi=run_wasi, - run_windows_msi=run_windows_msi, run_windows_tests=run_windows_tests, ) @@ -342,7 +329,6 @@ def process_target_branch(outputs: Outputs, git_branch: str) -> Outputs: if os.environ.get("GITHUB_EVENT_NAME", "").lower() == "workflow_dispatch": outputs.run_docs = True - outputs.run_windows_msi = True return outputs diff --git a/Tools/buildbot/buildmsi.bat b/Tools/buildbot/buildmsi.bat index 6804d794799509b..b0e196df2649be6 100644 --- a/Tools/buildbot/buildmsi.bat +++ b/Tools/buildbot/buildmsi.bat @@ -3,7 +3,9 @@ setlocal pushd -@rem build both snapshot MSIs -call "%~dp0..\msi\build.bat" -x86 -x64 +@rem The MSI project is gone, so instead we will just +@rem build and then do a default layout. +call "%~dp0..\..\PCbuild\build.bat" -e -k -v %* +call "%~dp0..\..\python.exe" "%~dp0..\..\PC\layout" --preset-default -o "%~dp0..\..\PCbuild\output" popd \ No newline at end of file diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 6e18593ad698570..ef314625d507d61 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -57,6 +57,9 @@ Python/pyhash.c - _Py_HashSecret - ## thread-safe hashtable (internal locks) Python/parking_lot.c - buckets - +## shared empty sentinel for the per-type method cache +Python/typecache.c - empty_cache - + ## data needed for introspecting asyncio state from debuggers and profilers Modules/_asynciomodule.c - _Py_AsyncioDebug - diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt index fa53b205c4ff6af..aeae9e6553a3aa6 100644 --- a/Tools/check-c-api-docs/ignored_c_api.txt +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -29,26 +29,12 @@ PY_DWORD_MAX PY_BIG_ENDIAN # cpython/methodobject.h PyCFunction_GET_CLASS -# cpython/compile.h -PyCF_ALLOW_INCOMPLETE_INPUT -PyCF_COMPILE_MASK -PyCF_DONT_IMPLY_DEDENT -PyCF_IGNORE_COOKIE -PyCF_MASK -PyCF_MASK_OBSOLETE -PyCF_SOURCE_IS_UTF8 # cpython/descrobject.h PyDescr_NAME PyDescr_TYPE PyWrapperFlag_KEYWORDS # cpython/fileobject.h Py_UniversalNewlineFgets -# cpython/pyframe.h -PyUnstable_EXECUTABLE_KINDS -PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION -PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR -PyUnstable_EXECUTABLE_KIND_PY_FUNCTION -PyUnstable_EXECUTABLE_KIND_SKIP # cpython/pylifecycle.h Py_FrozenMain # pythonrun.h diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index c8a914c22a9e137..a79242e740371ba 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -325,6 +325,19 @@ def enum_attr(): MyEnum.Y MyEnum.Z +_MCACHE_NUM_TYPES = 1 << 14 +_MCACHE_PAIRS = [ + (type(f"C{i}", (), {f"m{i}": i % 256})(), sys.intern(f"m{i}")) + for i in range(_MCACHE_NUM_TYPES) +] + +@register_benchmark +def type_lookup(): + pairs = _MCACHE_PAIRS + for _ in range(WORK_SCALE // 10): + for inst, name in pairs: + getattr(inst, name) + def bench_one_thread(func): t0 = time.perf_counter_ns() diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index ba52ea2a30e0be1..422e4f605920a0c 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -43,7 +43,6 @@ import gdb import os -import locale import sys @@ -107,8 +106,6 @@ def interp_frame_has_tlbc_index(): USED_TAGS = 0b11 -ENCODING = locale.getpreferredencoding() - FRAME_INFO_OPTIMIZED_OUT = '(frame information optimized out)' UNABLE_READ_INFO_PYTHON_FRAME = 'Unable to read information on python frame' EVALFRAME = '_PyEval_EvalFrameDefault' @@ -1504,6 +1501,10 @@ def proxyval(self, visited): def write_repr(self, out, visited): # Write this out as a Python str literal + # gdb writes its output in the host charset, so a character is escaped + # unless it is printable and encodable in that charset. + encoding = gdb.host_charset() + # Get a PyUnicodeObject* within the Python gdb process: proxy = self.proxyval(visited) @@ -1551,8 +1552,10 @@ def write_repr(self, out, visited): printable = ucs.isprintable() if printable: try: - ucs.encode(ENCODING) - except UnicodeEncodeError: + ucs.encode(encoding) + # LookupError or ValueError if the host charset is unknown + # or invalid. + except (UnicodeEncodeError, LookupError, ValueError): printable = False # Map Unicode whitespace and control characters diff --git a/Tools/msi/README.txt b/Tools/msi/README.txt deleted file mode 100644 index 4b3de9e82f6f1ac..000000000000000 --- a/Tools/msi/README.txt +++ /dev/null @@ -1,530 +0,0 @@ -Quick Build Info -================ - -For testing, the installer should be built with the Tools/msi/build.bat -script: - - build.bat [-x86] [-x64] [-ARM64] [--doc] - -For an official release, the installer should be built with the -Tools/msi/buildrelease.bat script and environment variables: - - set PYTHON= - set SPHINXBUILD= - set PATH=;%PATH% - - buildrelease.bat [-x86] [-x64] [-ARM64] [-D] [-B] - [-o ] [-c ] - -See the Building the Installer section for more information. - -Overview -======== - -Python is distributed on Windows as an installer that will configure the -user's system. This allows users to have a functioning copy of Python -without having to build it themselves. - -The main tasks of the installer are: - -* copy required files into the expected layout -* configure system settings so the installation can be located by - other programs -* add entry points for modifying, repairing and uninstalling Python -* make it easy to launch Python, its documentation, and IDLE - -Each of these is discussed in a later section of this document. - -Structure of the Installer -========================== - -The installer is structured as a 'layout', which consists of a number of -CAB and MSI files and a single EXE. - -The EXE is the main entry point into the installer. It contains the UI -and command-line logic, as well as the ability to locate and optionally -download other parts of the layout. - -Each MSI contains the logic required to install a component or feature -of Python. These MSIs should not be launched directly by users. MSIs can -be embedded into the EXE or automatically downloaded as needed. - -Each CAB contains the files making up a Python installation. CABs are -embedded into their associated MSI and are never seen by users. - -MSIs are only required when the related feature or component is being -installed. When components are not selected for installation, the -associated MSI is not downloaded. This allows the installer to offer -options to install debugging symbols and binaries without increasing -the initial download size by separating them into their own MSIs. - -Building the Installer -====================== - -Before building the installer, download the extra build dependencies -using Tools\msi\get_externals.bat. (Note that this is in addition to the -similarly named file in PCbuild.) - -One of the dependencies used in the build process is WiX, a toolset that -allows developers to create installers for Windows Installer, the -Windows installation engine. If you're not using the copy of WiX -installed by Tools\msi\get_externals.bat, you'll need to set the -"WixInstallPath" environment variable before building. - -Additionally, ensure that "MSVC v14x - VS 20xx C++ ARM64/ARM64EC build tools" -is selected under "Desktop Development with C++" in the "Visual Studio Installer", -even if you're not building on ARM64. This is required because we've upgraded -to WiX 3.14, which requires these tools for Python 3.10 and later versions. - -For testing, the installer should be built with the Tools/msi/build.bat -script: - - build.bat [-x86] [-x64] [-ARM64] [--doc] [--test-marker] [--pack] - -This script will build the required configurations of Python and -generate an installer layout in PCbuild/(win32|amd64)/en-us. - -Specify -x86, -x64 and/or -ARM64 to build for each platform. If none are -specified, both x64 and x86 will be built. Currently, both the debug and -release versions of Python are required for the installer. - -Specify --doc to include the documentation files. Ensure %PYTHON% and -%SPHINXBUILD% are set when passing this option. - -Specify --test-marker to build an installer that works side-by-side with -an official Python release. All registry keys and install locations will -include an extra marker to avoid overwriting files. This marker is -currently an 'x' prefix, but may change at any time. - -Specify --pack to build an installer that does not require all MSIs to -be available alongside. This takes longer, but is easier to share. - - -For an official release, the installer should be built with the -Tools/msi/buildrelease.bat script: - - set PYTHON= - set SPHINXBUILD= - set PATH=;%PATH% - - buildrelease.bat [-x86] [-x64] [-ARM64] [-D] [-B] - [-o ] [-c ] - -Specify -x86, -x64 and/or -ARM64 to build for each platform. If none are -specified, both x64 and x86 will be built. Currently, both the debug and -release versions of Python are required for the installer. - -Specify -D to skip rebuilding the documentation. The documentation is -required for a release and the build will fail if it is not available. -Ensure %PYTHON% and %SPHINXBUILD% are set if you omit this option. - -Specify -B to skip rebuilding Python. This is useful to only rebuild the -installer layout after a previous call to buildrelease.bat. - -Specify -o to set an output directory. The installer layouts will be -copied to platform-specific subdirectories of this path. - -Specify -c to choose a code-signing certificate to be used for all the -signable binaries in Python as well as each file making up the -installer. Official releases of Python must be signed. - - -If WiX is not found on your system, it will be automatically downloaded -and extracted to the externals/ directory. - -To manually build layouts of the installer, build one of the projects in -the bundle folder. - - msbuild bundle\snapshot.wixproj - msbuild bundle\releaseweb.wixproj - msbuild bundle\releaselocal.wixproj - msbuild bundle\full.wixproj - -snapshot.wixproj produces a test installer versioned based on the date. - -releaseweb.wixproj produces a release installer that does not embed any -of the layout. - -releaselocal.wixproj produces a release installer that embeds the files -required for a default installation. - -full.wixproj produces a test installer that embeds the entire layout. - -The following properties may be passed when building these projects. - - /p:BuildForRelease=(true|false) - When true, adds extra verification to ensure a complete installer is - produced. Defaults to false. - - /p:ReleaseUri=(any URI) - Used to generate unique IDs for the installers to allow side-by-side - installation. Forks of Python can use the same installer infrastructure - by providing a unique URI for this property. It does not need to be an - active internet address. Defaults to $(ComputerName). - - Official releases use https://www.python.org/(architecture name) - - /p:DownloadUrlBase=(any URI) - Specifies the base of a URL where missing parts of the installer layout - can be downloaded from. The build version and architecture will be - appended to create the full address. If omitted, missing components will - not be automatically downloaded. - - /p:DownloadUrl=(any URI) - Specifies the full URL where missing parts of the installer layout can - be downloaded from. Should normally include '{2}', which will be - substituted for the filename. If omitted, missing components will not be - automatically downloaded. If specified, this value overrides - DownloadUrlBase. - - /p:SigningCertificate=(certificate name) - Specifies the certificate to sign the installer layout with. If omitted, - the layout will not be signed. - - /p:RebuildAll=(true|false) - When true, rebuilds all of the MSIs making up the layout. Defaults to - true. - -Uploading the Installer -======================= - -For official releases, the uploadrelease.bat script should be used. - -You will require PuTTY so that plink.exe and pscp.exe can be used, and your -SSH key can be activated in pageant.exe. PuTTY should be either on your path -or in %ProgramFiles(x86)%\PuTTY. - -To include signatures for each uploaded file, you will need gpg2.exe on your -path or have run get_externals.bat. You may also need to "gpg2.exe --import" -your key before running the upload script. - - uploadrelease.bat --host --user [--dry-run] [--no-gpg] - -The host is the URL to the server. This can be provided by the Release -Manager. You should be able to SSH to this address. - -The username is your own username, which you have permission to SSH into -the server containing downloads. - -Use --dry-run to display the generated upload commands without executing -them. Signatures for each file will be generated but not uploaded unless ---no-gpg is also passed. - -Use --no-gpg to suppress signature generation and upload. - -The default target directory (which appears in uploadrelease.proj) is -correct for official Python releases, but may be overridden with ---target for other purposes. This path should generally not include -any version specifier, as that will be added automatically. - -Modifying the Installer -======================= - -The code for the installer is divided into three main groups: packages, -the bundle and the bootstrap application. - -Packages --------- - -Packages appear as subdirectories of Tools/msi (other than the bundle/ -directory). The project file is a .wixproj and the build output is a -single MSI. Packages are built with the WiX Toolset. Some project files -share source files and use preprocessor directives to enable particular -features. These are typically used to keep the sources close when the -files are related, but produce multiple independent packages. - -A package is the smallest element that may be independently installed or -uninstalled (as used in this installer). For example, the test suite has -its own package, as users can choose to add or remove it after the -initial installation. - -All the files installed by a single package should be related, though -some packages may not install any files. For example, the pip package -executes the ensurepip package, but does not add or remove any of its -own files. (It is represented as a package because of its -installed/uninstalled nature, as opposed to the "precompile standard -library" option, for example.) Dependencies between packages are handled -by the bundle, but packages should detect when dependencies are missing -and raise an error. - -Packages that include a lot of files may use an InstallFiles element in -the .wixproj file to generate sources. See lib/lib.wixproj for an -example, and msi.targets and csv_to_wxs.py for the implementation. This -element is also responsible for generating the code for cleaning up and -removing __pycache__ folders in any directory containing .py files. - -All packages are built with the Tools/msi/common.wxs file, and so any -directory or property in this file may be referenced. Of particular -interest: - - REGISTRYKEY (property) - The registry key for the current installation. - - InstallDirectory (directory) - The root install directory for the current installation. Subdirectories - are also specified in this file (DLLs, Lib, etc.) - - MenuDir (directory) - The Start Menu folder for the current installation. - - UpgradeTable (property) - Every package should reference this property to include upgrade - information. - - OptionalFeature (Component) - Packages that may be enabled or disabled should reference this component - and have an OPTIONAL_FEATURES entry in the bootstrap application to - properly handle Modify and Upgrade. - -The .wxl_template file is specially handled by the build system for this -project to perform {{substitutions}} as defined in msi.targets. They -should be included in projects as items, where .wxl files -are normally included as items. - -Bundle ------- - -The bundle is compiled to the main EXE entry point that for most users -will represent the Python installer. It is built from Tools/msi/bundle -with packages references in Tools/msi/bundle/packagegroups. - -Build logic for the bundle is in bundle.targets, but should be invoked -through one of the .wixproj files as described in Building the -Installer. - -The UI is separated between Default.thm (UI layout), Default.wxl -(strings), bundle.wxs (properties) and the bootstrap application. -Bundle.wxs also contains the chain, which is the list of packages to -install and the order they should be installed in. These refer to named -package groups in bundle/packagegroups. - -Each package group specifies one or more packages to install. Most -packages require two separate entries to support both per-user and -all-users installations. Because these reuse the same package, it does -not increase the overall size of the package. - -Package groups refer to payload groups, which allow better control over -embedding and downloading files than the default settings. Whether files -are embedded and where they are downloaded from depends on settings -created by the project files. - -Package references can include install conditions that determine when to -install the package. When a package is a dependency for others, the -condition should be crafted to ensure it is installed. - -MSI packages are installed or uninstalled based on their current state -and the install condition. This makes them most suitable for features -that are clearly present or absent from the user's machine. - -EXE packages are executed based on a customisable condition that can be -omitted. This makes them suitable for pre- or post-install tasks that -need to run regardless of whether features have been added or removed. - -Bootstrap Application ---------------------- - -The bootstrap application is a C++ application that controls the UI and -installation. While it does not directly compile into the main EXE of -the installer, it forms the main active component. Most of the -installation functionality is provided by WiX, and so the bootstrap -application is predominantly responsible for the code behind the UI that -is defined in the Default.thm file. The bootstrap application code is in -bundle/bootstrap and is built automatically when building the bundle. - -Installation Layout -=================== - -There are two installation layouts for Python on Windows, with the only -differences being supporting files. A layout is selected implicitly -based on whether the install is for all users of the machine or just for -the user performing the installation. - -The default installation location when installing for all users is -"%ProgramFiles%\Python3X" for the 64-bit interpreter and -"%ProgramFiles(x86)%\Python3X-32" for the 32-bit interpreter. (Note that -the latter path is equivalent to "%ProgramFiles%\Python3X-32" when -running a 32-bit version of Windows.) This location requires -administrative privileges to install or later modify the installation. - -The default installation location when installing for the current user -is "%LocalAppData%\Programs\Python\Python3X" for the 64-bit interpreter -and "%LocalAppData%\Programs\Python\Python3X-32" for the 32-bit -interpreter. Only the current user can access this location. This -provides a suitable level of protection against malicious modification -of Python's files. - -(Default installation locations are set in Tools\msi\bundle\bundle.wxs.) - -Within this install directory is the following approximate layout: - -.\python[w].exe The core executable files -.\python3x.dll The core interpreter -.\python3.dll The stable ABI reference -.\DLLs Stdlib extensions (*.pyd) and dependencies -.\Doc Documentation (*.html) -.\include Development headers (*.h) -.\Lib Standard library -.\Lib\test Test suite -.\libs Development libraries (*.lib) -.\Scripts Launcher scripts (*.exe, *.py) -.\tcl Tcl dependencies (*.dll, *.tcl and others) -.\Tools Tool scripts (*.py) - -When installed for all users, the following files are installed to -"%SystemRoot%" (typically "C:\Windows") to ensure they are always -available on PATH. (See Launching Python below.) For the current user, -they are installed in "%LocalAppData%\Programs\Python\PyLauncher". - -.\py[w].exe PEP 397 launcher - - -System Settings -=============== - -On installation, registry keys are created so that other applications -can locate and identify installations of Python. The locations of these -keys vary based on the install type. - -For 64-bit interpreters installed for all users, the root key is: - HKEY_LOCAL_MACHINE\Software\Python\PythonCore\3.X - -For 32-bit interpreters installed for all users on a 64-bit operating -system, the root key is: - HKEY_LOCAL_MACHINE\Software\Wow6432Node\Python\PythonCore\3.X-32 - -For 32-bit interpreters installed for all users on a 32-bit operating -system, the root key is: - HKEY_LOCAL_MACHINE\Software\Python\PythonCore\3.X-32 - -For 64-bit interpreters installed for the current user: - HKEY_CURRENT_USER\Software\Python\PythonCore\3.X - -For 32-bit interpreters installed for the current user: - HKEY_CURRENT_USER\Software\Python\PythonCore\3.X-32 - -When the core Python executables are installed, a key "InstallPath" is -created within the root key with its default value set to the -executable's install directory. A value named "ExecutablePath" is added -with the full path to the main Python interpreter, and a key -"InstallGroup" is created with its default value set to the product -name "Python 3.X". - -When the Python standard library is installed, a key "PythonPath" is -created within the root key with its default value set to the full path -to the Lib folder followed by the path to the DLLs folder, separated by -a semicolon. - -When the documentation is installed, a key "Help" is created within the -root key, with a subkey "Main Python Documentation" with its default -value set to the full path to the main index.html file. - - -The py.exe launcher is installed as part of a regular Python install, -but using a separate mechanism that allows it to more easily span -versions of Python. As a result, it has different root keys for its -registry entries: - -When installed for all users on a 64-bit operating system, the -launcher's root key is: - HKEY_LOCAL_MACHINE\Software\Wow6432Node\Python\Launcher - -When installed for all users on a 32-bit operating system, the -launcher's root key is: - HKEY_LOCAL_MACHINE\Software\Python\Launcher - -When installed for the current user: - HKEY_CURRENT_USER\Software\Python\Launcher - -When the launcher is installed, a key "InstallPath" is created within -its root key with its default value set to the launcher's install -directory. File associations are also created for .py, .pyw, .pyc and -.pyo files. - -Launching Python -================ - -When a feature offering user entry points in the Start Menu is -installed, a folder "Python 3.X" is created. Every shortcut should be -created within this folder, and each shortcut should include the version -and platform to allow users to identify the shortcut in a search results -page. - -The core Python executables creates a shortcut "Python 3.X (32-bit)" or -"Python 3.X (64-bit)" depending on the interpreter. - -The documentation creates a shortcut "Python 3.X 32-bit Manuals" or -"Python 3.X 64-bit Manuals". The documentation is identical for all -platforms, but the shortcuts need to be separate to avoid uninstallation -conflicts. - -Installing IDLE creates a shortcut "IDLE (Python 3.X 32-bit)" or "IDLE -(Python 3.X 64-bit)" depending on the interpreter. - - -For users who often launch Python from a Command Prompt, an option is -provided to add the directory containing python.exe to the user or -system PATH variable. If the option is selected, the install directory -and the Scripts directory will be added at the start of the system PATH -for an all users install and the user PATH for a per-user install. - -When the user only has one version of Python installed, this will behave -as expected. However, because Windows searches the system PATH before -the user PATH, users cannot override a system-wide installation of -Python on their PATH. Further, because the installer can only prepend to -the path, later installations of Python will take precedence over -earlier installations, regardless of interpreter version. - -Because it is not possible to automatically create a sensible PATH -configuration, users are recommended to use the py.exe launcher and -manually modify their PATH variable to add Scripts directories in their -preferred order. System-wide installations of Python should consider not -modifying PATH, or using an alternative technology to modify their -users' PATH variables. - - -The py.exe launcher is recommended because it uses a consistent and -sensible search order for Python installations. User installations are -preferred over system-wide installs, and later versions are preferred -regardless of installation order (with the exception that py.exe -currently prefers 2.x versions over 3.x versions without the -3 command -line argument). - -For both 32-bit and 64-bit interpreters, the 32-bit version of the -launcher is installed. This ensures that the search order is always -consistent (as the 64-bit launcher is subtly different from the 32-bit -launcher) and also avoids the need to install it multiple times. Future -versions of Python will upgrade the launcher in-place, using Windows -Installer's upgrade functionality to avoid conflicts with earlier -installed versions. - -When installed, file associations are created for .py, .pyc and .pyo -files to launch with py.exe and .pyw files to launch with pyw.exe. This -makes Python files respect shebang lines by default and also avoids -conflicts between multiple Python installations. - - -Repair, Modify and Uninstall -============================ - -After installation, Python may be modified, repaired or uninstalled by -running the original EXE again or via the Programs and Features applet -(formerly known as Add or Remove Programs). - -Modifications allow features to be added or removed. The install -directory and kind (all users/single user) cannot be modified. Because -Windows Installer caches installation packages, removing features will -not require internet access unless the package cache has been corrupted -or deleted. Adding features that were not previously installed and are -not embedded or otherwise available will require internet access. - -Repairing will rerun the installation for all currently installed -features, restoring files and registry keys that have been modified or -removed. This operation generally will not redownload any files unless -the cached packages have been corrupted or deleted. - -Removing Python will clean up all the files and registry keys that were -created by the installer, as well as __pycache__ folders that are -explicitly handled by the installer. Python packages installed later -using a tool like pip will not be removed. Some components may be -installed by other installers and these will not be removed if another -product has a dependency on them. diff --git a/Tools/msi/appendpath/appendpath.wixproj b/Tools/msi/appendpath/appendpath.wixproj deleted file mode 100644 index 897087678c56f63..000000000000000 --- a/Tools/msi/appendpath/appendpath.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {12B59A06-37CC-4558-A9C8-DAE922E64EF3} - 2.0 - appendpath - Package - ICE71 - - - - - - - - - - - diff --git a/Tools/msi/appendpath/appendpath.wxs b/Tools/msi/appendpath/appendpath.wxs deleted file mode 100644 index bba186cf60fd10a..000000000000000 --- a/Tools/msi/appendpath/appendpath.wxs +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - NOT ALLUSERS=1 - - - - - - - - - - - - ALLUSERS=1 - - - - - - - - - - - - - - - - diff --git a/Tools/msi/appendpath/appendpath_en-US.wxl b/Tools/msi/appendpath/appendpath_en-US.wxl deleted file mode 100644 index 19a2e7734f8c31e..000000000000000 --- a/Tools/msi/appendpath/appendpath_en-US.wxl +++ /dev/null @@ -1,6 +0,0 @@ - - - Append to Path - AppendPath - No !(loc.ProductName) installation was detected. - diff --git a/Tools/msi/build.bat b/Tools/msi/build.bat deleted file mode 100644 index 2fe8a475e7e3a30..000000000000000 --- a/Tools/msi/build.bat +++ /dev/null @@ -1,103 +0,0 @@ -@echo off -setlocal -set D=%~dp0 -set PCBUILD=%D%..\..\PCbuild\ - -set BUILDX86= -set BUILDX64= -set BUILDARM64= -set BUILDDOC= -set BUILDTEST= -set BUILDPACK= -set REBUILD= - -:CheckOpts -if "%~1" EQU "-h" goto Help -if /I "%~1" EQU "-x86" (set BUILDX86=1) && shift && goto CheckOpts -if /I "%~1" EQU "-Win32" (set BUILDX86=1) && shift && goto CheckOpts -if /I "%~1" EQU "-x64" (set BUILDX64=1) && shift && goto CheckOpts -if /I "%~1" EQU "-arm64" (set BUILDARM64=1) && shift && goto CheckOpts -if "%~1" EQU "--doc" (set BUILDDOC=1) && shift && goto CheckOpts -if "%~1" EQU "--no-test-marker" (set BUILDTEST=) && shift && goto CheckOpts -if "%~1" EQU "--test-marker" (set BUILDTEST=--test-marker) && shift && goto CheckOpts -if "%~1" EQU "--pack" (set BUILDPACK=1) && shift && goto CheckOpts -if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts -rem %IncludeFreethreaded% is recognised by the MSI build, but not the regular build. -rem We use it to build twice and then build the installer with its extra option -if /I "%~1" EQU "--disable-gil" (set IncludeFreethreaded=true) && shift && goto CheckOpts - -if not defined BUILDX86 if not defined BUILDX64 if not defined BUILDARM64 (set BUILDX86=1) && (set BUILDX64=1) - -call "%D%get_externals.bat" -call "%PCBUILD%find_msbuild.bat" %MSBUILD% -if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) - -if defined BUILDX86 call "%PCBUILD%build.bat" -p Win32 -d -e %REBUILD% %BUILDTEST% -if errorlevel 1 exit /B %ERRORLEVEL% -if defined BUILDX86 call "%PCBUILD%build.bat" -p Win32 -e %REBUILD% %BUILDTEST% -if errorlevel 1 exit /B %ERRORLEVEL% - -if defined BUILDX64 call "%PCBUILD%build.bat" -p x64 -d -e %REBUILD% %BUILDTEST% -if errorlevel 1 exit /B %ERRORLEVEL% -if defined BUILDX64 call "%PCBUILD%build.bat" -p x64 -e %REBUILD% %BUILDTEST% -if errorlevel 1 exit /B %ERRORLEVEL% - -if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -d -e %REBUILD% %BUILDTEST% -if errorlevel 1 exit /B %ERRORLEVEL% -if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -e %REBUILD% %BUILDTEST% -if errorlevel 1 exit /B %ERRORLEVEL% - -if /I "%IncludeFreethreaded%"=="true" ( - rem Cannot "exit /B" inside an if block because %ERRORLEVEL% will be wrong. - rem We just skip everything after the first "errorlevel 1" and then exit after - if defined BUILDX86 call "%PCBUILD%build.bat" -p Win32 -d -e %REBUILD% %BUILDTEST% --disable-gil - if not errorlevel 1 if defined BUILDX86 call "%PCBUILD%build.bat" -p Win32 -e %REBUILD% %BUILDTEST% --disable-gil - - if not errorlevel 1 if defined BUILDX64 call "%PCBUILD%build.bat" -p x64 -d -e %REBUILD% %BUILDTEST% --disable-gil - if not errorlevel 1 if defined BUILDX64 call "%PCBUILD%build.bat" -p x64 -e %REBUILD% %BUILDTEST% --disable-gil - - if not errorlevel 1 if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -d -e %REBUILD% %BUILDTEST% --disable-gil - if not errorlevel 1 if defined BUILDARM64 call "%PCBUILD%build.bat" -p ARM64 -e %REBUILD% %BUILDTEST% --disable-gil -) -if errorlevel 1 exit /B %ERRORLEVEL% - -if defined BUILDDOC call "%PCBUILD%..\Doc\make.bat" html -if errorlevel 1 exit /B %ERRORLEVEL% - -rem Build the launcher MSI separately -%MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 -if errorlevel 1 exit /B %ERRORLEVEL% - -set BUILD_CMD="%D%bundle\snapshot.wixproj" -if defined BUILDTEST ( - set BUILD_CMD=%BUILD_CMD% /p:UseTestMarker=true -) -if defined BUILDPACK ( - set BUILD_CMD=%BUILD_CMD% /p:Pack=true -) -if defined REBUILD ( - set BUILD_CMD=%BUILD_CMD% /t:Rebuild -) - -if defined BUILDX86 %MSBUILD% /p:Platform=x86 %BUILD_CMD% -if errorlevel 1 exit /B %ERRORLEVEL% - -if defined BUILDX64 %MSBUILD% /p:Platform=x64 %BUILD_CMD% -if errorlevel 1 exit /B %ERRORLEVEL% - -if defined BUILDARM64 %MSBUILD% /p:Platform=ARM64 %BUILD_CMD% -if errorlevel 1 exit /B %ERRORLEVEL% - -exit /B 0 - -:Help -echo build.bat [-x86] [-x64] [-arm64] [--doc] [-h] [--test-marker] [--pack] [-r] -echo. -echo -x86 Build x86 installers -echo -x64 Build x64 installers -echo -ARM64 Build ARM64 installers -echo --doc Build documentation -echo --test-marker Build with test markers -echo --no-test-marker Build without test markers (default) -echo --pack Embed core MSIs into installer -echo -r Rebuild rather than incremental build diff --git a/Tools/msi/buildrelease.bat b/Tools/msi/buildrelease.bat deleted file mode 100644 index 77fb47792088531..000000000000000 --- a/Tools/msi/buildrelease.bat +++ /dev/null @@ -1,261 +0,0 @@ -@setlocal -@echo off - -rem This script is intended for building official releases of Python. -rem To use it to build alternative releases, you should clone this file -rem and modify the following three URIs. - -rem These two will ensure that your release can be installed -rem alongside an official Python release, by modifying the GUIDs used -rem for all components. -rem -rem The following substitutions will be applied to the release URI: -rem Variable Description Example -rem {arch} architecture amd64, win32 -rem Do not change the scheme to https. Otherwise, releases built with this -rem script will not be upgradable to/from official releases of Python. -set RELEASE_URI=http://www.python.org/{arch} - -rem This is the URL that will be used to download installation files. -rem The files available from the default URL *will* conflict with your -rem installer. Trust me, you don't want them, even if it seems like a -rem good idea. -rem -rem The following substitutions will be applied to the download URL: -rem Variable Description Example -rem {version} version number 3.5.0 -rem {arch} architecture amd64, win32 -rem {releasename} release name a1, b2, rc3 (or blank for final) -rem {msi} MSI filename core.msi -set DOWNLOAD_URL=https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} - -set D=%~dp0 -set PCBUILD=%D%..\..\PCbuild\ -if NOT DEFINED Py_OutDir set Py_OutDir=%PCBUILD% -set EXTERNALS=%D%..\..\externals\windows-installer\ - -set BUILDX86= -set BUILDX64= -set BUILDARM64= -set TARGET=Rebuild -set TESTTARGETDIR= -set PGO=-m test -q --pgo -set BUILDMSI=1 -set BUILDNUGET=1 -set BUILDZIP=1 - - -:CheckOpts -if "%1" EQU "-h" goto Help -if "%1" EQU "-c" (set CERTNAME=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--certificate" (set CERTNAME=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-o" (set OUTDIR=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--out" (set OUTDIR=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-D" (set SKIPDOC=1) && shift && goto CheckOpts -if "%1" EQU "--skip-doc" (set SKIPDOC=1) && shift && goto CheckOpts -if "%1" EQU "-B" (set SKIPBUILD=1) && shift && goto CheckOpts -if "%1" EQU "--skip-build" (set SKIPBUILD=1) && shift && goto CheckOpts -if "%1" EQU "--download" (set DOWNLOAD_URL=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--test" (set TESTTARGETDIR=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-b" (set TARGET=Build) && shift && goto CheckOpts -if "%1" EQU "--build" (set TARGET=Build) && shift && goto CheckOpts -if /I "%1" EQU "-x86" (set BUILDX86=1) && shift && goto CheckOpts -if /I "%1" EQU "-Win32" (set BUILDX86=1) && shift && goto CheckOpts -if /I "%1" EQU "-x64" (set BUILDX64=1) && shift && goto CheckOpts -if /I "%1" EQU "-arm64" (set BUILDARM64=1) && shift && goto CheckOpts -if "%1" EQU "--pgo" (set PGO=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--skip-pgo" (set PGO=) && shift && goto CheckOpts -if "%1" EQU "--skip-nuget" (set BUILDNUGET=) && shift && goto CheckOpts -if "%1" EQU "--skip-zip" (set BUILDZIP=) && shift && goto CheckOpts -if "%1" EQU "--skip-msi" (set BUILDMSI=) && shift && goto CheckOpts - -if "%1" NEQ "" echo Invalid option: "%1" && exit /B 1 - -if not defined BUILDX86 if not defined BUILDX64 if not defined BUILDARM64 (set BUILDX86=1) && (set BUILDX64=1) - -if not exist "%GIT%" where git > "%TEMP%\git.loc" 2> nul && set /P GIT= < "%TEMP%\git.loc" & del "%TEMP%\git.loc" -if not exist "%GIT%" echo Cannot find Git on PATH && exit /B 1 - -call "%D%get_externals.bat" -call "%PCBUILD%find_msbuild.bat" %MSBUILD% -if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) - -:builddoc -if "%SKIPBUILD%" EQU "1" goto skipdoc -if "%SKIPDOC%" EQU "1" goto skipdoc - -call "%D%..\..\doc\make.bat" html -if errorlevel 1 exit /B %ERRORLEVEL% -:skipdoc - -if defined BUILDX86 ( - call :build x86 - if errorlevel 1 exit /B %ERRORLEVEL% -) - -if defined BUILDX64 ( - call :build x64 "%PGO%" - if errorlevel 1 exit /B %ERRORLEVEL% -) - -if defined BUILDARM64 ( - call :build ARM64 - if errorlevel 1 exit /B %ERRORLEVEL% -) - -if defined TESTTARGETDIR ( - call "%D%testrelease.bat" -t "%TESTTARGETDIR%" - if errorlevel 1 exit /B %ERRORLEVEL% -) - -exit /B 0 - -:build -@setlocal -@echo off - -if "%1" EQU "x86" ( - set PGO= - set BUILD=%Py_OutDir%win32\ - set BUILD_PLAT=Win32 - set OUTDIR_PLAT=win32 - set OBJDIR_PLAT=x86 -) else if "%1" EQU "x64" ( - set BUILD=%Py_OutDir%amd64\ - set PGO=%~2 - set BUILD_PLAT=x64 - set OUTDIR_PLAT=amd64 - set OBJDIR_PLAT=x64 -) else if "%1" EQU "ARM64" ( - set BUILD=%Py_OutDir%arm64\ - set PGO=%~2 - set BUILD_PLAT=ARM64 - set OUTDIR_PLAT=arm64 - set OBJDIR_PLAT=arm64 -) else ( - echo Unknown platform %1 - exit /B 1 -) - -if exist "%BUILD%en-us" ( - echo Deleting %BUILD%en-us - rmdir /q/s "%BUILD%en-us" - if errorlevel 1 exit /B %ERRORLEVEL% -) - -if exist "%D%obj\Debug_%OBJDIR_PLAT%" ( - echo Deleting "%D%obj\Debug_%OBJDIR_PLAT%" - rmdir /q/s "%D%obj\Debug_%OBJDIR_PLAT%" - if errorlevel 1 exit /B %ERRORLEVEL% -) - -if exist "%D%obj\Release_%OBJDIR_PLAT%" ( - echo Deleting "%D%obj\Release_%OBJDIR_PLAT%" - rmdir /q/s "%D%obj\Release_%OBJDIR_PLAT%" - if errorlevel 1 exit /B %ERRORLEVEL% -) - -if not "%CERTNAME%" EQU "" ( - set CERTOPTS="/p:SigningCertificate=%CERTNAME%" -) else ( - set CERTOPTS= -) -if not "%PGO%" EQU "" ( - set PGOOPTS=--pgo-job "%PGO%" -) else ( - set PGOOPTS= -) -if not "%SKIPBUILD%" EQU "1" ( - @echo call "%PCBUILD%build.bat" -e -p %BUILD_PLAT% -t %TARGET% %PGOOPTS% %CERTOPTS% - @call "%PCBUILD%build.bat" -e -p %BUILD_PLAT% -t %TARGET% %PGOOPTS% %CERTOPTS% - @if errorlevel 1 exit /B %ERRORLEVEL% - @rem build.bat turns echo back on, so we disable it again - @echo off - - @echo call "%PCBUILD%build.bat" -d -e -p %BUILD_PLAT% -t %TARGET% - @call "%PCBUILD%build.bat" -d -e -p %BUILD_PLAT% -t %TARGET% - @if errorlevel 1 exit /B %ERRORLEVEL% - @rem build.bat turns echo back on, so we disable it again - @echo off -) - -if "%OUTDIR_PLAT%" EQU "win32" ( - %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% - if errorlevel 1 exit /B %ERRORLEVEL% -) else if not exist "%Py_OutDir%win32\en-us\launcher.msi" ( - %MSBUILD% "%D%launcher\launcher.wixproj" /p:Platform=x86 %CERTOPTS% /p:ReleaseUri=%RELEASE_URI% - if errorlevel 1 exit /B %ERRORLEVEL% -) - -set BUILDOPTS=/p:Platform=%1 /p:BuildForRelease=true /p:DownloadUrl=%DOWNLOAD_URL% /p:DownloadUrlBase=%DOWNLOAD_URL_BASE% /p:ReleaseUri=%RELEASE_URI% -if defined BUILDMSI ( - %MSBUILD% "%D%bundle\releaselocal.wixproj" /t:Rebuild %BUILDOPTS% %CERTOPTS% /p:RebuildAll=true - if errorlevel 1 exit /B %ERRORLEVEL% -) - -if defined BUILDZIP ( - if "%BUILD_PLAT%" EQU "ARM64" ( - echo Skipping embeddable ZIP generation for ARM64 platform - ) else ( - %MSBUILD% "%D%make_zip.proj" /t:Build %BUILDOPTS% %CERTOPTS% /p:OutputPath="%BUILD%en-us" - if errorlevel 1 exit /B %ERRORLEVEL% - ) -) - -if defined BUILDNUGET ( - if "%BUILD_PLAT%" EQU "ARM64" ( - echo Skipping Nuget package generation for ARM64 platform - ) else ( - %MSBUILD% "%D%..\nuget\make_pkg.proj" /t:Build /p:Configuration=Release /p:Platform=%1 /p:OutputPath="%BUILD%en-us" - if errorlevel 1 exit /B %ERRORLEVEL% - ) -) - -if not "%OUTDIR%" EQU "" ( - mkdir "%OUTDIR%\%OUTDIR_PLAT%" - mkdir "%OUTDIR%\%OUTDIR_PLAT%\binaries" - mkdir "%OUTDIR%\%OUTDIR_PLAT%\symbols" - robocopy "%BUILD%en-us" "%OUTDIR%\%OUTDIR_PLAT%" /XF "*.wixpdb" - robocopy "%BUILD%\" "%OUTDIR%\%OUTDIR_PLAT%\binaries" *.exe *.dll *.pyd /XF "_test*" /XF "*_d.*" /XF "_freeze*" /XF "tcl*" /XF "tk*" /XF "*_test.*" - robocopy "%BUILD%\" "%OUTDIR%\%OUTDIR_PLAT%\symbols" *.pdb /XF "_test*" /XF "*_d.*" /XF "_freeze*" /XF "tcl*" /XF "tk*" /XF "*_test.*" -) - -exit /B 0 - -:Help -echo buildrelease.bat [--out DIR] [-x86] [-x64] [-arm64] [--certificate CERTNAME] [--build] [--pgo COMMAND] -echo [--skip-build] [--skip-doc] [--skip-nuget] [--skip-zip] [--skip-pgo] -echo [--download DOWNLOAD URL] [--test TARGETDIR] -echo [-h] -echo. -echo --out (-o) Specify an additional output directory for installers -echo -x86 Build x86 installers -echo -x64 Build x64 installers -echo -arm64 Build ARM64 installers -echo --build (-b) Incrementally build Python rather than rebuilding -echo --skip-build (-B) Do not build Python (just do the installers) -echo --skip-doc (-D) Do not build documentation -echo --pgo Specify PGO command for x64 installers -echo --skip-pgo Build x64 installers without using PGO -echo --skip-msi Do not build executable/MSI packages -echo --skip-nuget Do not build Nuget packages -echo --skip-zip Do not build embeddable package -echo --download Specify the full download URL for MSIs -echo --test Specify the test directory to run the installer tests -echo -h Display this help information -echo. -echo If no architecture is specified, all architectures will be built. -echo If --test is not specified, the installer tests are not run. -echo. -echo For the --pgo option, any Python command line can be used, or 'default' to -echo use the default task (-m test --pgo). -echo. -echo x86 and ARM64 builds will never use PGO. ARM64 builds will never generate -echo embeddable or Nuget packages. -echo. -echo The following substitutions will be applied to the download URL: -echo Variable Description Example -echo {version} version number 3.5.0 -echo {arch} architecture amd64, win32 -echo {releasename} release name a1, b2, rc3 (or blank for final) -echo {msi} MSI filename core.msi diff --git a/Tools/msi/bundle/Default.ARM64.xsl b/Tools/msi/bundle/Default.ARM64.xsl deleted file mode 100644 index b28b3f264adbc2d..000000000000000 --- a/Tools/msi/bundle/Default.ARM64.xsl +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/Default.thm b/Tools/msi/bundle/Default.thm deleted file mode 100644 index 6e908ac373442d9..000000000000000 --- a/Tools/msi/bundle/Default.thm +++ /dev/null @@ -1,141 +0,0 @@ - - - #(loc.Caption) - Segoe UI - Segoe UI - Segoe UI - Segoe UI - Segoe UI - Segoe UI - Segoe UI - - - #(loc.HelpHeader) - - - #(loc.HelpText) - - - - #(loc.InstallHeader) - - - #(loc.InstallMessage) - - #(loc.DeprecationMessage) - - - - - #(loc.ShortInstallLauncherAllUsersLabel) - #(loc.ShortPrependPathLabel) - - - - - #(loc.InstallUpgradeHeader) - - - #(loc.InstallUpgradeMessage) - - - - - - - - #(loc.InstallHeader) - - - - - - - - #(loc.Custom1Header) - - - #(loc.Include_docLabel) - #(loc.Include_docHelpLabel) - - #(loc.Include_pipLabel) - #(loc.Include_pipHelpLabel) - - #(loc.Include_tcltkLabel) - #(loc.Include_tcltkHelpLabel) - - #(loc.Include_testLabel) - #(loc.Include_testHelpLabel) - - #(loc.Include_launcherLabel) - #(loc.InstallLauncherAllUsersLabel) - - - - - - - - #(loc.Custom2Header) - - - #(loc.InstallAllUsersLabel) - #(loc.AssociateFilesLabel) - #(loc.ShortcutsLabel) - #(loc.PrependPathLabel) - #(loc.PrecompileLabel) - #(loc.Include_symbolsLabel) - #(loc.Include_debugLabel) - #(loc.Include_freethreadedLabel) - - #(loc.CustomLocationLabel) - - - #(loc.CustomLocationHelpLabel) - - - - - - - #(loc.ProgressHeader) - - - #(loc.ProgressLabel) - #(loc.OverallProgressPackageText) - - - - - #(loc.ModifyHeader) - - - - - - - - - - #(loc.SuccessHeader) - - - - - - - #(loc.SuccessRestartText) - - - - - #(loc.FailureHeader) - - - #(loc.FailureHyperlinkLogText) - - #(loc.FailureRestartText) - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/Default.wxl b/Tools/msi/bundle/Default.wxl deleted file mode 100644 index 335c1d922d97b52..000000000000000 --- a/Tools/msi/bundle/Default.wxl +++ /dev/null @@ -1,133 +0,0 @@ - - - [WixBundleName] Setup - Installing - Setup - Updating - Modify - Repairing - Repair - Removing - Uninstall - - You will be prompted for Administrator privileges to install a C Runtime Library update (KB2999226). - - -Continue? - - &Cancel - &Close - Install [WixBundleName] - Select Install Now to install Python with default settings, or choose Customize to enable or disable features. - Version [WixBundleVersion] - Upgrade to [WixBundleName] - Select Upgrade Now to keep your current settings, or choose Customize to enable or disable features. - Are you sure you want to cancel? - Previous version - Setup Help - Visit <a href="http://docs.python.org/[ShortVersion]/using/windows.html">docs.python.org/[ShortVersion]/using/windows.html</a> for the full list of options, including the ability to enable and disable specific features. - -"/passive" to display progress without requiring user interaction - -"/quiet" to install/uninstall without displaying any UI - -"/simple" to prevent user customization - -"/uninstall" to remove Python (without confirmation) - -"/layout [\[]directory[\]]" to pre-download all components - -"/log [\[]filename[\]]" to specify log files location - [WixBundleName] <a href="#">license terms</a>. - I &agree to the license terms and conditions - &Install Now - [TargetDir] - -Includes IDLE, pip and documentation -Creates shortcuts and file associations - C&ustomize installation - Choose location and features - &Install - Use settings preselected by your administrator - -[SimpleInstallDescription] - Up&grade Now - [TargetDir] - -Replaces your existing installation without changing settings. -Select Customize to review current options. - C&ustomize installation - Choose location and features - Optional Features - Advanced Options - Customize install location - You will require write permissions for the selected location. - &Install - &Next - &Back - B&rowse - &Documentation - Installs the Python documentation files. - &pip - Installs pip, which can download and install other Python packages. - Tcl/Tk, turtle and &IDLE - Installs tkinter, turtle and the IDLE development environment. - Python &test suite - Installs the standard library test suite. - py &launcher - Installs the global 'py' launcher to make it easier to start Python. - Use Programs and Features to remove the 'py' launcher. - Upgrades the global 'py' launcher from the previous version. - - Associate &files with Python (requires the 'py' launcher) - Create shortcuts for installed applications - Add Python to &environment variables - Add &python.exe to PATH - Append Python to &environment variables - Append &python.exe to PATH - Install Python [ShortVersion] for &all users - for &all users (requires admin privileges) - Use admin privi&leges when installing py.exe - Python Launcher is already installed - &Precompile standard library - Download debugging &symbols - Download debu&g binaries (requires VS 2017 or later) - Download &free-threaded binaries - - [ActionLikeInstallation] Progress - [ActionLikeInstalling]: - Initializing... - Modify Setup - &Modify - Add or remove individual features. - &Repair - Ensure all current features are correctly installed. - &Uninstall - Remove the entire [WixBundleName] installation. - [ActionLikeInstallation] was successful - &Launch - You may need to restart your computer to finish updating files. - &Restart - New to Python? Start with the <a href="https://docs.python.org/[ShortVersion]/tutorial/index.html">online tutorial</a> and <a href="https://docs.python.org/[ShortVersion]/index.html">documentation</a>. At your terminal, type "py" to launch Python, or search for Python in your Start menu. - -See <a href="https://docs.python.org/[ShortVersion]/whatsnew/[ShortVersion].html">what's new</a> in this release, or find more info about <a href="https://docs.python.org/[ShortVersion]/using/windows.html">using Python on Windows</a>. - Thank you for using [WixBundleName]. - Thank you for using [WixBundleName]. - -Feel free to post at <a href="https://discuss.python.org/c/users/7">discuss.python.org</a> if you continue to encounter issues. - Thank you for using [WixBundleName]. - -Feel free to post at <a href="https://discuss.python.org/c/users/7">discuss.python.org</a> if you encountered problems. - Setup failed - One or more issues caused the setup to fail. Please fix the issues and then retry setup. For more information see the <a href="#">log file</a>. - You must restart your computer to complete the rollback of the software. - &Restart - Unable to install [WixBundleName] due to an existing install. Use Programs and Features to modify, repair or remove [WixBundleName]. - At least Windows 10 or Windows Server 2016 are required to install [WixBundleName] - -Visit <a href="https://www.python.org/downloads/">python.org</a> to download an earlier version of Python. - Disable path length limit - Changes your machine configuration to allow programs, including Python, to bypass the 260 character "MAX_PATH" limitation. - - NOTE: This installer is being retired and will no longer be available after Python 3.15. <a href="https://docs.python.org/using/windows">More info</a> - diff --git a/Tools/msi/bundle/SideBar.png b/Tools/msi/bundle/SideBar.png deleted file mode 100644 index 52f15dd46c18fe5..000000000000000 Binary files a/Tools/msi/bundle/SideBar.png and /dev/null differ diff --git a/Tools/msi/bundle/bootstrap/LICENSE.txt b/Tools/msi/bundle/bootstrap/LICENSE.txt deleted file mode 100644 index 5791a7e81d74ff8..000000000000000 --- a/Tools/msi/bundle/bootstrap/LICENSE.txt +++ /dev/null @@ -1,25 +0,0 @@ -This license applies to the bootstrapper application that is embedded within the installer. It has no impact on the licensing for the rest of the installer or Python itself, as no code covered by this license exists in any other part of the product. - ---- - -Microsoft Reciprocal License (MS-RL) - -This license governs use of the accompanying software. If you use the software, you accept this license. If you do not accept the license, do not use the software. - -1. Definitions - The terms "reproduce," "reproduction," "derivative works," and "distribution" have the same meaning here as under U.S. copyright law. - A "contribution" is the original software, or any additions or changes to the software. - A "contributor" is any person that distributes its contribution under this license. - "Licensed patents" are a contributor's patent claims that read directly on its contribution. - -2. Grant of Rights - (A) Copyright Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free copyright license to reproduce its contribution, prepare derivative works of its contribution, and distribute its contribution or any derivative works that you create. - (B) Patent Grant- Subject to the terms of this license, including the license conditions and limitations in section 3, each contributor grants you a non-exclusive, worldwide, royalty-free license under its licensed patents to make, have made, use, sell, offer for sale, import, and/or otherwise dispose of its contribution in the software or derivative works of the contribution in the software. - -3. Conditions and Limitations - (A) Reciprocal Grants- For any file you distribute that contains code from the software (in source code or binary format), you must provide recipients the source code to that file along with a copy of this license, which license will govern that file. You may license other files that are entirely your own work and do not contain code from the software under any terms you choose. - (B) No Trademark License- This license does not grant you rights to use any contributors' name, logo, or trademarks. - (C) If you bring a patent claim against any contributor over patents that you claim are infringed by the software, your patent license from such contributor to the software ends automatically. - (D) If you distribute any portion of the software, you must retain all copyright, patent, trademark, and attribution notices that are present in the software. - (E) If you distribute any portion of the software in source code form, you may do so only under this license by including a complete copy of this license with your distribution. If you distribute any portion of the software in compiled or object code form, you may only do so under a license that complies with this license. - (F) The software is licensed "as-is." You bear the risk of using it. The contributors give no express warranties, guarantees or conditions. You may have additional consumer rights under your local laws which this license cannot change. To the extent permitted under your local laws, the contributors exclude the implied warranties of merchantability, fitness for a particular purpose and non-infringement. diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp deleted file mode 100644 index 1e0df5084ff0795..000000000000000 --- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp +++ /dev/null @@ -1,3323 +0,0 @@ -//------------------------------------------------------------------------------------------------- -// -// Copyright (c) 2004, Outercurve Foundation. -// This software is released under Microsoft Reciprocal License (MS-RL). -// The license and further copyright text can be found in the file -// LICENSE.TXT at the root directory of the distribution. -// -//------------------------------------------------------------------------------------------------- - - -#include "pch.h" - -static const LPCWSTR PYBA_WINDOW_CLASS = L"PythonBA"; -static const DWORD PYBA_ACQUIRE_PERCENTAGE = 30; -static const LPCWSTR PYBA_VARIABLE_BUNDLE_FILE_VERSION = L"WixBundleFileVersion"; - -enum PYBA_STATE { - PYBA_STATE_INITIALIZING, - PYBA_STATE_INITIALIZED, - PYBA_STATE_HELP, - PYBA_STATE_DETECTING, - PYBA_STATE_DETECTED, - PYBA_STATE_PLANNING, - PYBA_STATE_PLANNED, - PYBA_STATE_APPLYING, - PYBA_STATE_CACHING, - PYBA_STATE_CACHED, - PYBA_STATE_EXECUTING, - PYBA_STATE_EXECUTED, - PYBA_STATE_APPLIED, - PYBA_STATE_FAILED, -}; - -static const int WM_PYBA_SHOW_HELP = WM_APP + 100; -static const int WM_PYBA_DETECT_PACKAGES = WM_APP + 101; -static const int WM_PYBA_PLAN_PACKAGES = WM_APP + 102; -static const int WM_PYBA_APPLY_PACKAGES = WM_APP + 103; -static const int WM_PYBA_CHANGE_STATE = WM_APP + 104; -static const int WM_PYBA_SHOW_FAILURE = WM_APP + 105; - -// This enum must be kept in the same order as the PAGE_NAMES array. -enum PAGE { - PAGE_LOADING, - PAGE_HELP, - PAGE_INSTALL, - PAGE_UPGRADE, - PAGE_SIMPLE_INSTALL, - PAGE_CUSTOM1, - PAGE_CUSTOM2, - PAGE_MODIFY, - PAGE_PROGRESS, - PAGE_PROGRESS_PASSIVE, - PAGE_SUCCESS, - PAGE_FAILURE, - COUNT_PAGE, -}; - -// This array must be kept in the same order as the PAGE enum. -static LPCWSTR PAGE_NAMES[] = { - L"Loading", - L"Help", - L"Install", - L"Upgrade", - L"SimpleInstall", - L"Custom1", - L"Custom2", - L"Modify", - L"Progress", - L"ProgressPassive", - L"Success", - L"Failure", -}; - -enum CONTROL_ID { - // Non-paged controls - ID_CLOSE_BUTTON = THEME_FIRST_ASSIGN_CONTROL_ID, - ID_MINIMIZE_BUTTON, - - // Welcome page - ID_INSTALL_BUTTON, - ID_INSTALL_CUSTOM_BUTTON, - ID_INSTALL_SIMPLE_BUTTON, - ID_INSTALL_UPGRADE_BUTTON, - ID_INSTALL_UPGRADE_CUSTOM_BUTTON, - ID_INSTALL_CANCEL_BUTTON, - ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, - - // Customize Page - ID_TARGETDIR_EDITBOX, - ID_CUSTOM_ASSOCIATE_FILES_CHECKBOX, - ID_CUSTOM_INSTALL_ALL_USERS_CHECKBOX, - ID_CUSTOM_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, - ID_CUSTOM_INCLUDE_LAUNCHER_HELP_LABEL, - ID_CUSTOM_COMPILE_ALL_CHECKBOX, - ID_CUSTOM_BROWSE_BUTTON, - ID_CUSTOM_BROWSE_BUTTON_LABEL, - ID_CUSTOM_INSTALL_BUTTON, - ID_CUSTOM_NEXT_BUTTON, - ID_CUSTOM1_BACK_BUTTON, - ID_CUSTOM2_BACK_BUTTON, - ID_CUSTOM1_CANCEL_BUTTON, - ID_CUSTOM2_CANCEL_BUTTON, - - // Modify page - ID_MODIFY_BUTTON, - ID_REPAIR_BUTTON, - ID_UNINSTALL_BUTTON, - ID_MODIFY_CANCEL_BUTTON, - - // Progress page - ID_CACHE_PROGRESS_PACKAGE_TEXT, - ID_CACHE_PROGRESS_BAR, - ID_CACHE_PROGRESS_TEXT, - - ID_EXECUTE_PROGRESS_PACKAGE_TEXT, - ID_EXECUTE_PROGRESS_BAR, - ID_EXECUTE_PROGRESS_TEXT, - ID_EXECUTE_PROGRESS_ACTIONDATA_TEXT, - - ID_OVERALL_PROGRESS_PACKAGE_TEXT, - ID_OVERALL_PROGRESS_BAR, - ID_OVERALL_CALCULATED_PROGRESS_BAR, - ID_OVERALL_PROGRESS_TEXT, - - ID_PROGRESS_CANCEL_BUTTON, - - // Success page - ID_SUCCESS_TEXT, - ID_SUCCESS_RESTART_TEXT, - ID_SUCCESS_RESTART_BUTTON, - ID_SUCCESS_CANCEL_BUTTON, - ID_SUCCESS_MAX_PATH_BUTTON, - - // Failure page - ID_FAILURE_LOGFILE_LINK, - ID_FAILURE_MESSAGE_TEXT, - ID_FAILURE_RESTART_TEXT, - ID_FAILURE_RESTART_BUTTON, - ID_FAILURE_CANCEL_BUTTON -}; - -static THEME_ASSIGN_CONTROL_ID CONTROL_ID_NAMES[] = { - { ID_CLOSE_BUTTON, L"CloseButton" }, - { ID_MINIMIZE_BUTTON, L"MinimizeButton" }, - - { ID_INSTALL_BUTTON, L"InstallButton" }, - { ID_INSTALL_CUSTOM_BUTTON, L"InstallCustomButton" }, - { ID_INSTALL_SIMPLE_BUTTON, L"InstallSimpleButton" }, - { ID_INSTALL_UPGRADE_BUTTON, L"InstallUpgradeButton" }, - { ID_INSTALL_UPGRADE_CUSTOM_BUTTON, L"InstallUpgradeCustomButton" }, - { ID_INSTALL_CANCEL_BUTTON, L"InstallCancelButton" }, - { ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, L"InstallLauncherAllUsers" }, - - { ID_TARGETDIR_EDITBOX, L"TargetDir" }, - { ID_CUSTOM_ASSOCIATE_FILES_CHECKBOX, L"AssociateFiles" }, - { ID_CUSTOM_INSTALL_ALL_USERS_CHECKBOX, L"InstallAllUsers" }, - { ID_CUSTOM_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, L"CustomInstallLauncherAllUsers" }, - { ID_CUSTOM_INCLUDE_LAUNCHER_HELP_LABEL, L"Include_launcherHelp" }, - { ID_CUSTOM_COMPILE_ALL_CHECKBOX, L"CompileAll" }, - { ID_CUSTOM_BROWSE_BUTTON, L"CustomBrowseButton" }, - { ID_CUSTOM_BROWSE_BUTTON_LABEL, L"CustomBrowseButtonLabel" }, - { ID_CUSTOM_INSTALL_BUTTON, L"CustomInstallButton" }, - { ID_CUSTOM_NEXT_BUTTON, L"CustomNextButton" }, - { ID_CUSTOM1_BACK_BUTTON, L"Custom1BackButton" }, - { ID_CUSTOM2_BACK_BUTTON, L"Custom2BackButton" }, - { ID_CUSTOM1_CANCEL_BUTTON, L"Custom1CancelButton" }, - { ID_CUSTOM2_CANCEL_BUTTON, L"Custom2CancelButton" }, - - { ID_MODIFY_BUTTON, L"ModifyButton" }, - { ID_REPAIR_BUTTON, L"RepairButton" }, - { ID_UNINSTALL_BUTTON, L"UninstallButton" }, - { ID_MODIFY_CANCEL_BUTTON, L"ModifyCancelButton" }, - - { ID_CACHE_PROGRESS_PACKAGE_TEXT, L"CacheProgressPackageText" }, - { ID_CACHE_PROGRESS_BAR, L"CacheProgressbar" }, - { ID_CACHE_PROGRESS_TEXT, L"CacheProgressText" }, - { ID_EXECUTE_PROGRESS_PACKAGE_TEXT, L"ExecuteProgressPackageText" }, - { ID_EXECUTE_PROGRESS_BAR, L"ExecuteProgressbar" }, - { ID_EXECUTE_PROGRESS_TEXT, L"ExecuteProgressText" }, - { ID_EXECUTE_PROGRESS_ACTIONDATA_TEXT, L"ExecuteProgressActionDataText" }, - { ID_OVERALL_PROGRESS_PACKAGE_TEXT, L"OverallProgressPackageText" }, - { ID_OVERALL_PROGRESS_BAR, L"OverallProgressbar" }, - { ID_OVERALL_CALCULATED_PROGRESS_BAR, L"OverallCalculatedProgressbar" }, - { ID_OVERALL_PROGRESS_TEXT, L"OverallProgressText" }, - { ID_PROGRESS_CANCEL_BUTTON, L"ProgressCancelButton" }, - - { ID_SUCCESS_TEXT, L"SuccessText" }, - { ID_SUCCESS_RESTART_TEXT, L"SuccessRestartText" }, - { ID_SUCCESS_RESTART_BUTTON, L"SuccessRestartButton" }, - { ID_SUCCESS_CANCEL_BUTTON, L"SuccessCancelButton" }, - { ID_SUCCESS_MAX_PATH_BUTTON, L"SuccessMaxPathButton" }, - - { ID_FAILURE_LOGFILE_LINK, L"FailureLogFileLink" }, - { ID_FAILURE_MESSAGE_TEXT, L"FailureMessageText" }, - { ID_FAILURE_RESTART_TEXT, L"FailureRestartText" }, - { ID_FAILURE_RESTART_BUTTON, L"FailureRestartButton" }, - { ID_FAILURE_CANCEL_BUTTON, L"FailureCancelButton" }, -}; - -static struct { LPCWSTR regName; LPCWSTR variableName; } OPTIONAL_FEATURES[] = { - { L"core_d", L"Include_debug" }, - { L"core_pdb", L"Include_symbols" }, - { L"dev", L"Include_dev" }, - { L"doc", L"Include_doc" }, - { L"exe", L"Include_exe" }, - { L"lib", L"Include_lib" }, - { L"path", L"PrependPath" }, - { L"appendpath", L"AppendPath" }, - { L"pip", L"Include_pip" }, - { L"tcltk", L"Include_tcltk" }, - { L"test", L"Include_test" }, - { L"tools", L"Include_tools" }, - { L"Shortcuts", L"Shortcuts" }, - // Include_launcher and AssociateFiles are handled separately and so do - // not need to be included in this list. - { L"freethreaded", L"Include_freethreaded" }, - { nullptr, nullptr } -}; - - - -class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { - void ShowPage(DWORD newPageId) { - // Process each control for special handling in the new page. - ProcessPageControls(ThemeGetPage(_theme, newPageId)); - - // Enable disable controls per-page. - if (_pageIds[PAGE_INSTALL] == newPageId || - _pageIds[PAGE_SIMPLE_INSTALL] == newPageId || - _pageIds[PAGE_UPGRADE] == newPageId) { - InstallPage_Show(); - } else if (_pageIds[PAGE_CUSTOM1] == newPageId) { - Custom1Page_Show(); - } else if (_pageIds[PAGE_CUSTOM2] == newPageId) { - Custom2Page_Show(); - } else if (_pageIds[PAGE_MODIFY] == newPageId) { - ModifyPage_Show(); - } else if (_pageIds[PAGE_SUCCESS] == newPageId) { - SuccessPage_Show(); - } else if (_pageIds[PAGE_FAILURE] == newPageId) { - FailurePage_Show(); - } - - // Prevent repainting while switching page to avoid ugly flickering - _suppressPaint = TRUE; - ThemeShowPage(_theme, newPageId, SW_SHOW); - ThemeShowPage(_theme, _visiblePageId, SW_HIDE); - _suppressPaint = FALSE; - InvalidateRect(_theme->hwndParent, nullptr, TRUE); - _visiblePageId = newPageId; - - // On the install page set the focus to the install button or - // the next enabled control if install is disabled - if (_pageIds[PAGE_INSTALL] == newPageId) { - ThemeSetFocus(_theme, ID_INSTALL_BUTTON); - } else if (_pageIds[PAGE_SIMPLE_INSTALL] == newPageId) { - ThemeSetFocus(_theme, ID_INSTALL_SIMPLE_BUTTON); - } - } - - // - // Handles control clicks - // - void OnCommand(CONTROL_ID id) { - LPWSTR defaultDir = nullptr; - LPWSTR targetDir = nullptr; - LONGLONG elevated, crtInstalled, installAllUsers; - BOOL checked, launcherChecked; - WCHAR wzPath[MAX_PATH] = { }; - BROWSEINFOW browseInfo = { }; - PIDLIST_ABSOLUTE pidl = nullptr; - DWORD pageId; - HRESULT hr = S_OK; - - switch(id) { - case ID_CLOSE_BUTTON: - OnClickCloseButton(); - break; - - // Install commands - case ID_INSTALL_SIMPLE_BUTTON: __fallthrough; - case ID_INSTALL_UPGRADE_BUTTON: __fallthrough; - case ID_INSTALL_BUTTON: - SavePageSettings(); - - hr = BalGetNumericVariable(L"InstallAllUsers", &installAllUsers); - ExitOnFailure(hr, L"Failed to get install scope"); - - hr = _engine->SetVariableNumeric(L"CompileAll", installAllUsers); - ExitOnFailure(hr, L"Failed to update CompileAll"); - - hr = EnsureTargetDir(); - ExitOnFailure(hr, L"Failed to set TargetDir"); - - OnPlan(BOOTSTRAPPER_ACTION_INSTALL); - break; - - case ID_CUSTOM1_BACK_BUTTON: - SavePageSettings(); - if (_modifying) { - GoToPage(PAGE_MODIFY); - } else if (_upgrading) { - GoToPage(PAGE_UPGRADE); - } else { - GoToPage(PAGE_INSTALL); - } - break; - - case ID_INSTALL_CUSTOM_BUTTON: __fallthrough; - case ID_INSTALL_UPGRADE_CUSTOM_BUTTON: __fallthrough; - case ID_CUSTOM2_BACK_BUTTON: - SavePageSettings(); - GoToPage(PAGE_CUSTOM1); - break; - - case ID_CUSTOM_NEXT_BUTTON: - SavePageSettings(); - GoToPage(PAGE_CUSTOM2); - break; - - case ID_CUSTOM_INSTALL_BUTTON: - SavePageSettings(); - - hr = EnsureTargetDir(); - ExitOnFailure(hr, L"Failed to set TargetDir"); - - hr = BalGetStringVariable(L"TargetDir", &targetDir); - if (SUCCEEDED(hr)) { - // TODO: Check whether directory exists and contains another installation - ReleaseStr(targetDir); - } - - OnPlan(_command.action); - break; - - case ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX: - checked = ThemeIsControlChecked(_theme, ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX); - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", checked); - - ThemeControlElevates(_theme, ID_INSTALL_BUTTON, WillElevate()); - break; - - case ID_CUSTOM_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX: - checked = ThemeIsControlChecked(_theme, ID_CUSTOM_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX); - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", checked); - - ThemeControlElevates(_theme, ID_CUSTOM_INSTALL_BUTTON, WillElevate()); - break; - - case ID_CUSTOM_INSTALL_ALL_USERS_CHECKBOX: - checked = ThemeIsControlChecked(_theme, ID_CUSTOM_INSTALL_ALL_USERS_CHECKBOX); - _engine->SetVariableNumeric(L"InstallAllUsers", checked); - - ThemeControlElevates(_theme, ID_CUSTOM_INSTALL_BUTTON, WillElevate()); - ThemeControlEnable(_theme, ID_CUSTOM_BROWSE_BUTTON_LABEL, !checked); - if (checked) { - _engine->SetVariableNumeric(L"CompileAll", 1); - ThemeSendControlMessage(_theme, ID_CUSTOM_COMPILE_ALL_CHECKBOX, BM_SETCHECK, BST_CHECKED, 0); - } - ThemeGetTextControl(_theme, ID_TARGETDIR_EDITBOX, &targetDir); - if (targetDir) { - // Check the current value against the default to see - // if we should switch it automatically. - hr = BalGetStringVariable( - checked ? L"DefaultJustForMeTargetDir" : L"DefaultAllUsersTargetDir", - &defaultDir - ); - - if (SUCCEEDED(hr) && defaultDir) { - LPWSTR formatted = nullptr; - if (defaultDir[0] && SUCCEEDED(BalFormatString(defaultDir, &formatted))) { - if (wcscmp(formatted, targetDir) == 0) { - ReleaseStr(defaultDir); - defaultDir = nullptr; - ReleaseStr(formatted); - formatted = nullptr; - - hr = BalGetStringVariable( - checked ? L"DefaultAllUsersTargetDir" : L"DefaultJustForMeTargetDir", - &defaultDir - ); - if (SUCCEEDED(hr) && defaultDir && defaultDir[0] && SUCCEEDED(BalFormatString(defaultDir, &formatted))) { - ThemeSetTextControl(_theme, ID_TARGETDIR_EDITBOX, formatted); - ReleaseStr(formatted); - } - } else { - ReleaseStr(formatted); - } - } - - ReleaseStr(defaultDir); - } - } - break; - - case ID_CUSTOM_BROWSE_BUTTON: - browseInfo.hwndOwner = _hWnd; - browseInfo.pszDisplayName = wzPath; - browseInfo.lpszTitle = _theme->sczCaption; - browseInfo.ulFlags = BIF_RETURNONLYFSDIRS | BIF_USENEWUI; - pidl = ::SHBrowseForFolderW(&browseInfo); - if (pidl && ::SHGetPathFromIDListW(pidl, wzPath)) { - ThemeSetTextControl(_theme, ID_TARGETDIR_EDITBOX, wzPath); - } - - if (pidl) { - ::CoTaskMemFree(pidl); - } - break; - - // Modify commands - case ID_MODIFY_BUTTON: - // Some variables cannot be modified - _engine->SetVariableString(L"InstallAllUsersState", L"disable"); - _engine->SetVariableString(L"InstallLauncherAllUsersState", L"disable"); - _engine->SetVariableString(L"TargetDirState", L"disable"); - _engine->SetVariableString(L"CustomBrowseButtonState", L"disable"); - _modifying = TRUE; - GoToPage(PAGE_CUSTOM1); - break; - - case ID_REPAIR_BUTTON: - OnPlan(BOOTSTRAPPER_ACTION_REPAIR); - break; - - case ID_UNINSTALL_BUTTON: - OnPlan(BOOTSTRAPPER_ACTION_UNINSTALL); - break; - - case ID_SUCCESS_MAX_PATH_BUTTON: - EnableMaxPathSupport(); - ThemeControlEnable(_theme, ID_SUCCESS_MAX_PATH_BUTTON, FALSE); - break; - } - - LExit: - return; - } - - void InstallPage_Show() { - // Ensure the All Users install button has a UAC shield - BOOL elevated = WillElevate(); - ThemeControlElevates(_theme, ID_INSTALL_BUTTON, elevated); - ThemeControlElevates(_theme, ID_INSTALL_SIMPLE_BUTTON, elevated); - ThemeControlElevates(_theme, ID_INSTALL_UPGRADE_BUTTON, elevated); - - LONGLONG blockedLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) && blockedLauncher) { - LOC_STRING *pLocString = nullptr; - if (SUCCEEDED(LocGetString(_wixLoc, L"#(loc.ShortInstallLauncherBlockedLabel)", &pLocString)) && pLocString) { - ThemeSetTextControl(_theme, ID_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, pLocString->wzText); - } - } - } - - void Custom1Page_Show() { - LONGLONG installLauncherAllUsers; - - if (FAILED(BalGetNumericVariable(L"InstallLauncherAllUsers", &installLauncherAllUsers))) { - installLauncherAllUsers = 0; - } - - ThemeSendControlMessage(_theme, ID_CUSTOM_INSTALL_LAUNCHER_ALL_USERS_CHECKBOX, BM_SETCHECK, - installLauncherAllUsers ? BST_CHECKED : BST_UNCHECKED, 0); - - LOC_STRING *pLocString = nullptr; - LPCWSTR locKey = L"#(loc.Include_launcherHelp)"; - LONGLONG blockedLauncher; - - if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) && blockedLauncher) { - locKey = L"#(loc.Include_launcherRemove)"; - } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &blockedLauncher)) && blockedLauncher) { - locKey = L"#(loc.Include_launcherUpgrade)"; - } - - if (SUCCEEDED(LocGetString(_wixLoc, locKey, &pLocString)) && pLocString) { - ThemeSetTextControl(_theme, ID_CUSTOM_INCLUDE_LAUNCHER_HELP_LABEL, pLocString->wzText); - } - } - - void Custom2Page_Show() { - HRESULT hr; - LONGLONG installAll, includeLauncher; - - if (FAILED(BalGetNumericVariable(L"InstallAllUsers", &installAll))) { - installAll = 0; - } - - if (WillElevate()) { - ThemeControlElevates(_theme, ID_CUSTOM_INSTALL_BUTTON, TRUE); - ThemeShowControl(_theme, ID_CUSTOM_BROWSE_BUTTON_LABEL, SW_HIDE); - } else { - ThemeControlElevates(_theme, ID_CUSTOM_INSTALL_BUTTON, FALSE); - ThemeShowControl(_theme, ID_CUSTOM_BROWSE_BUTTON_LABEL, SW_SHOW); - } - - if (SUCCEEDED(BalGetNumericVariable(L"Include_launcher", &includeLauncher)) && includeLauncher) { - ThemeControlEnable(_theme, ID_CUSTOM_ASSOCIATE_FILES_CHECKBOX, TRUE); - } else { - ThemeSendControlMessage(_theme, ID_CUSTOM_ASSOCIATE_FILES_CHECKBOX, BM_SETCHECK, BST_UNCHECKED, 0); - ThemeControlEnable(_theme, ID_CUSTOM_ASSOCIATE_FILES_CHECKBOX, FALSE); - } - - LPWSTR targetDir = nullptr; - hr = BalGetStringVariable(L"TargetDir", &targetDir); - if (SUCCEEDED(hr) && targetDir && targetDir[0]) { - ThemeSetTextControl(_theme, ID_TARGETDIR_EDITBOX, targetDir); - StrFree(targetDir); - } else if (SUCCEEDED(hr)) { - StrFree(targetDir); - targetDir = nullptr; - - LPWSTR defaultTargetDir = nullptr; - hr = BalGetStringVariable(L"DefaultCustomTargetDir", &defaultTargetDir); - if (SUCCEEDED(hr) && defaultTargetDir && !defaultTargetDir[0]) { - StrFree(defaultTargetDir); - defaultTargetDir = nullptr; - - hr = BalGetStringVariable( - installAll ? L"DefaultAllUsersTargetDir" : L"DefaultJustForMeTargetDir", - &defaultTargetDir - ); - } - if (SUCCEEDED(hr) && defaultTargetDir) { - if (defaultTargetDir[0] && SUCCEEDED(BalFormatString(defaultTargetDir, &targetDir))) { - ThemeSetTextControl(_theme, ID_TARGETDIR_EDITBOX, targetDir); - StrFree(targetDir); - } - StrFree(defaultTargetDir); - } - } - } - - void ModifyPage_Show() { - ThemeControlEnable(_theme, ID_REPAIR_BUTTON, !_suppressRepair); - } - - void SuccessPage_Show() { - // on the "Success" page, check if the restart button should be enabled. - BOOL showRestartButton = FALSE; - LOC_STRING *successText = nullptr; - HRESULT hr = S_OK; - - if (_restartRequired) { - if (BOOTSTRAPPER_RESTART_PROMPT == _command.restart) { - showRestartButton = TRUE; - } - } - - switch (_plannedAction) { - case BOOTSTRAPPER_ACTION_INSTALL: - hr = LocGetString(_wixLoc, L"#(loc.SuccessInstallMessage)", &successText); - break; - case BOOTSTRAPPER_ACTION_MODIFY: - hr = LocGetString(_wixLoc, L"#(loc.SuccessModifyMessage)", &successText); - break; - case BOOTSTRAPPER_ACTION_REPAIR: - hr = LocGetString(_wixLoc, L"#(loc.SuccessRepairMessage)", &successText); - break; - case BOOTSTRAPPER_ACTION_UNINSTALL: - hr = LocGetString(_wixLoc, L"#(loc.SuccessRemoveMessage)", &successText); - break; - } - - if (successText) { - LPWSTR formattedString = nullptr; - BalFormatString(successText->wzText, &formattedString); - if (formattedString) { - ThemeSetTextControl(_theme, ID_SUCCESS_TEXT, formattedString); - StrFree(formattedString); - } - } - - ThemeControlEnable(_theme, ID_SUCCESS_RESTART_TEXT, showRestartButton); - ThemeControlEnable(_theme, ID_SUCCESS_RESTART_BUTTON, showRestartButton); - - if (_command.action != BOOTSTRAPPER_ACTION_INSTALL || - !IsWindowsVersionOrGreater(10, 0, 0)) { - ThemeControlEnable(_theme, ID_SUCCESS_MAX_PATH_BUTTON, FALSE); - } else { - DWORD dataType = 0, buffer = 0, bufferLen = sizeof(buffer); - HKEY hKey; - LRESULT res = RegOpenKeyExW( - HKEY_LOCAL_MACHINE, - L"SYSTEM\\CurrentControlSet\\Control\\FileSystem", - 0, - KEY_READ, - &hKey - ); - if (res == ERROR_SUCCESS) { - res = RegQueryValueExW(hKey, L"LongPathsEnabled", nullptr, &dataType, - (LPBYTE)&buffer, &bufferLen); - RegCloseKey(hKey); - } - else { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Failed to open SYSTEM\\CurrentControlSet\\Control\\FileSystem: error code %d", res); - } - if (res == ERROR_SUCCESS && dataType == REG_DWORD && buffer == 0) { - ThemeControlElevates(_theme, ID_SUCCESS_MAX_PATH_BUTTON, TRUE); - } - else { - if (res == ERROR_SUCCESS) - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Failed to read LongPathsEnabled value: error code %d", res); - else - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Hiding MAX_PATH button because it is already enabled"); - ThemeControlEnable(_theme, ID_SUCCESS_MAX_PATH_BUTTON, FALSE); - } - } - } - - void FailurePage_Show() { - // on the "Failure" page, show error message and check if the restart button should be enabled. - - // if there is a log file variable then we'll assume the log file exists. - BOOL showLogLink = (_bundle.sczLogVariable && *_bundle.sczLogVariable); - BOOL showErrorMessage = FALSE; - BOOL showRestartButton = FALSE; - - if (FAILED(_hrFinal)) { - LPWSTR unformattedText = nullptr; - LPWSTR text = nullptr; - - // If we know the failure message, use that. - if (_failedMessage && *_failedMessage) { - StrAllocString(&unformattedText, _failedMessage, 0); - } else { - // try to get the error message from the error code. - StrAllocFromError(&unformattedText, _hrFinal, nullptr); - if (!unformattedText || !*unformattedText) { - StrAllocFromError(&unformattedText, E_FAIL, nullptr); - } - } - - if (E_WIXSTDBA_CONDITION_FAILED == _hrFinal) { - if (unformattedText) { - StrAllocString(&text, unformattedText, 0); - } - } else { - StrAllocFormatted(&text, L"0x%08x - %ls", _hrFinal, unformattedText); - } - - if (text) { - ThemeSetTextControl(_theme, ID_FAILURE_MESSAGE_TEXT, text); - showErrorMessage = TRUE; - } - - ReleaseStr(text); - ReleaseStr(unformattedText); - } - - if (_restartRequired && BOOTSTRAPPER_RESTART_PROMPT == _command.restart) { - showRestartButton = TRUE; - } - - ThemeControlEnable(_theme, ID_FAILURE_LOGFILE_LINK, showLogLink); - ThemeControlEnable(_theme, ID_FAILURE_MESSAGE_TEXT, showErrorMessage); - ThemeControlEnable(_theme, ID_FAILURE_RESTART_TEXT, showRestartButton); - ThemeControlEnable(_theme, ID_FAILURE_RESTART_BUTTON, showRestartButton); - } - - static void EnableMaxPathSupport() { - LPWSTR targetDir = nullptr, defaultDir = nullptr; - HRESULT hr = BalGetStringVariable(L"TargetDir", &targetDir); - if (FAILED(hr) || !targetDir || !targetDir[0]) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to get TargetDir"); - return; - } - - LPWSTR pythonw = nullptr; - StrAllocFormatted(&pythonw, L"%ls\\pythonw.exe", targetDir); - if (!pythonw || !pythonw[0]) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to construct pythonw.exe path"); - return; - } - - LPCWSTR arguments = L"-c \"import winreg; " - "winreg.SetValueEx(" - "winreg.CreateKey(winreg.HKEY_LOCAL_MACHINE, " - "r'SYSTEM\\CurrentControlSet\\Control\\FileSystem'), " - "'LongPathsEnabled', " - "None, " - "winreg.REG_DWORD, " - "1" - ")\""; - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Executing %ls %ls", pythonw, arguments); - HINSTANCE res = ShellExecuteW(0, L"runas", pythonw, arguments, NULL, SW_HIDE); - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "return code 0x%08x", res); - } - -public: // IBootstrapperApplication - virtual STDMETHODIMP OnStartup() { - HRESULT hr = S_OK; - DWORD dwUIThreadId = 0; - - // create UI thread - _hUiThread = ::CreateThread(nullptr, 0, UiThreadProc, this, 0, &dwUIThreadId); - if (!_hUiThread) { - ExitWithLastError(hr, "Failed to create UI thread."); - } - - LExit: - return hr; - } - - - virtual STDMETHODIMP_(int) OnShutdown() { - int nResult = IDNOACTION; - - // wait for UI thread to terminate - if (_hUiThread) { - ::WaitForSingleObject(_hUiThread, INFINITE); - ReleaseHandle(_hUiThread); - } - - // If a restart was required. - if (_restartRequired && _allowRestart) { - nResult = IDRESTART; - } - - return nResult; - } - - virtual STDMETHODIMP_(int) OnDetectRelatedMsiPackage( - __in_z LPCWSTR wzPackageId, - __in_z LPCWSTR /*wzProductCode*/, - __in BOOL fPerMachine, - __in DWORD64 /*dw64Version*/, - __in BOOTSTRAPPER_RELATED_OPERATION operation - ) { - // Only check launcher_AllUsers because we'll find the same packages - // twice if we check launcher_JustForMe as well. - if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, L"launcher_AllUsers", -1)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Detected existing launcher install"); - - LONGLONG blockedLauncher, detectedLauncher; - if (FAILED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher))) { - blockedLauncher = 0; - } - - // Get the prior DetectedLauncher value so we can see if we've - // detected more than one, and then update the stored variable - // (we use the original value later on via the local). - if (FAILED(BalGetNumericVariable(L"DetectedLauncher", &detectedLauncher))) { - detectedLauncher = 0; - } - if (!detectedLauncher) { - _engine->SetVariableNumeric(L"DetectedLauncher", 1); - } - - if (blockedLauncher) { - // Nothing else to do, we're already blocking - } - else if (BOOTSTRAPPER_RELATED_OPERATION_DOWNGRADE == operation) { - // Found a higher version, so we can't install ours. - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Higher version launcher has been detected."); - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Launcher will not be installed"); - _engine->SetVariableNumeric(L"BlockedLauncher", 1); - } - else if (detectedLauncher) { - if (!blockedLauncher) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Multiple launcher installs have been detected."); - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "No launcher will be installed or upgraded until one has been removed."); - _engine->SetVariableNumeric(L"BlockedLauncher", 1); - } - } - else if (BOOTSTRAPPER_RELATED_OPERATION_MAJOR_UPGRADE == operation) { - // Found an older version, so let's run the equivalent as an upgrade - // This overrides "unknown" all users options, but will leave alone - // any that have already been set/detected. - // User can deselect the option to include the launcher, but cannot - // change it from the current per user/machine setting. - LONGLONG includeLauncher, includeLauncherAllUsers; - if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))) { - includeLauncher = -1; - } - if (FAILED(BalGetNumericVariable(L"InstallLauncherAllUsers", &includeLauncherAllUsers))) { - includeLauncherAllUsers = -1; - } - - if (includeLauncher < 0) { - _engine->SetVariableNumeric(L"Include_launcher", 1); - } - if (includeLauncherAllUsers < 0) { - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", fPerMachine ? 1 : 0); - } else if (includeLauncherAllUsers != fPerMachine ? 1 : 0) { - // Requested AllUsers option is inconsistent, so block - _engine->SetVariableNumeric(L"BlockedLauncher", 1); - } - _engine->SetVariableNumeric(L"DetectedOldLauncher", 1); - } - } - return CheckCanceled() ? IDCANCEL : IDNOACTION; - } - - virtual STDMETHODIMP_(int) OnDetectRelatedBundle( - __in LPCWSTR wzBundleId, - __in BOOTSTRAPPER_RELATION_TYPE relationType, - __in LPCWSTR /*wzBundleTag*/, - __in BOOL fPerMachine, - __in DWORD64 /*dw64Version*/, - __in BOOTSTRAPPER_RELATED_OPERATION operation - ) { - BalInfoAddRelatedBundleAsPackage(&_bundle.packages, wzBundleId, relationType, fPerMachine); - - // Remember when our bundle would cause a downgrade. - if (BOOTSTRAPPER_RELATED_OPERATION_DOWNGRADE == operation) { - _downgradingOtherVersion = TRUE; - } else if (BOOTSTRAPPER_RELATED_OPERATION_MAJOR_UPGRADE == operation) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Detected previous version - planning upgrade"); - _upgrading = TRUE; - - LoadOptionalFeatureStates(_engine); - } else if (BOOTSTRAPPER_RELATED_OPERATION_NONE == operation) { - if (_command.action == BOOTSTRAPPER_ACTION_INSTALL) { - LOC_STRING *pLocString = nullptr; - if (SUCCEEDED(LocGetString(_wixLoc, L"#(loc.FailureExistingInstall)", &pLocString)) && pLocString) { - BalFormatString(pLocString->wzText, &_failedMessage); - } else { - BalFormatString(L"Cannot install [WixBundleName] because it is already installed.", &_failedMessage); - } - BalLog( - BOOTSTRAPPER_LOG_LEVEL_ERROR, - "Related bundle %ls is preventing install", - wzBundleId - ); - SetState(PYBA_STATE_FAILED, E_WIXSTDBA_CONDITION_FAILED); - } - } - - return CheckCanceled() ? IDCANCEL : IDOK; - } - - - virtual STDMETHODIMP_(void) OnDetectPackageComplete( - __in LPCWSTR wzPackageId, - __in HRESULT hrStatus, - __in BOOTSTRAPPER_PACKAGE_STATE state - ) { } - - - virtual STDMETHODIMP_(void) OnDetectComplete(__in HRESULT hrStatus) { - if (SUCCEEDED(hrStatus) && _baFunction) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Running detect complete BA function"); - _baFunction->OnDetectComplete(); - } - - if (SUCCEEDED(hrStatus)) { - // Update launcher install states - // If we didn't detect any existing installs, Include_launcher and - // InstallLauncherAllUsers will both be -1, so we will set to their - // defaults and leave the options enabled. - // Otherwise, if we detected an existing install, we disable the - // options so they remain fixed. - // The code in OnDetectRelatedMsiPackage is responsible for figuring - // out whether existing installs are compatible with the settings in - // place during detection. - LONGLONG blockedLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) - && blockedLauncher) { - _engine->SetVariableNumeric(L"Include_launcher", 0); - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", 0); - _engine->SetVariableString(L"InstallLauncherAllUsersState", L"disable"); - _engine->SetVariableString(L"Include_launcherState", L"disable"); - } - else { - LONGLONG includeLauncher, includeLauncherAllUsers, associateFiles; - - if (FAILED(BalGetNumericVariable(L"Include_launcher", &includeLauncher))) { - includeLauncher = -1; - } - if (FAILED(BalGetNumericVariable(L"InstallLauncherAllUsers", &includeLauncherAllUsers))) { - includeLauncherAllUsers = -1; - } - if (FAILED(BalGetNumericVariable(L"AssociateFiles", &associateFiles))) { - associateFiles = -1; - } - - if (includeLauncherAllUsers < 0) { - includeLauncherAllUsers = 0; - _engine->SetVariableNumeric(L"InstallLauncherAllUsers", includeLauncherAllUsers); - } - - if (includeLauncher < 0) { - if (BOOTSTRAPPER_ACTION_LAYOUT == _command.action || - (BOOTSTRAPPER_ACTION_INSTALL == _command.action && !_upgrading)) { - // When installing/downloading, we include the launcher - // (though downloads should ignore this setting anyway) - _engine->SetVariableNumeric(L"Include_launcher", 1); - } else { - // Any other action, we should have detected an existing - // install (e.g. on remove/modify), so if we didn't, we - // assume it's not selected. - _engine->SetVariableNumeric(L"Include_launcher", 0); - _engine->SetVariableNumeric(L"AssociateFiles", 0); - } - } - - if (associateFiles < 0) { - auto hr = LoadAssociateFilesStateFromKey( - _engine, - includeLauncherAllUsers ? HKEY_LOCAL_MACHINE : HKEY_CURRENT_USER - ); - if (FAILED(hr)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Failed to load AssociateFiles state: error code 0x%08X", hr); - } else if (hr == S_OK) { - associateFiles = 1; - } - _engine->SetVariableNumeric(L"AssociateFiles", associateFiles); - } - } - } - - if (SUCCEEDED(hrStatus)) { - hrStatus = EvaluateConditions(); - } - - if (SUCCEEDED(hrStatus)) { - // Ensure the default path has been set - hrStatus = EnsureTargetDir(); - } - - SetState(PYBA_STATE_DETECTED, hrStatus); - - // If we're not interacting with the user or we're doing a layout or we're just after a force restart - // then automatically start planning. - if (BOOTSTRAPPER_DISPLAY_FULL > _command.display || - BOOTSTRAPPER_ACTION_LAYOUT == _command.action || - BOOTSTRAPPER_ACTION_UNINSTALL == _command.action || - BOOTSTRAPPER_RESUME_TYPE_REBOOT == _command.resumeType) { - if (SUCCEEDED(hrStatus)) { - ::PostMessageW(_hWnd, WM_PYBA_PLAN_PACKAGES, 0, _command.action); - } - } - } - - - virtual STDMETHODIMP_(int) OnPlanRelatedBundle( - __in_z LPCWSTR /*wzBundleId*/, - __inout_z BOOTSTRAPPER_REQUEST_STATE* pRequestedState - ) { - return CheckCanceled() ? IDCANCEL : IDOK; - } - - - virtual STDMETHODIMP_(int) OnPlanPackageBegin( - __in_z LPCWSTR wzPackageId, - __inout BOOTSTRAPPER_REQUEST_STATE *pRequestState - ) { - HRESULT hr = S_OK; - BAL_INFO_PACKAGE* pPackage = nullptr; - - if (_nextPackageAfterRestart) { - // After restart we need to finish the dependency registration for our package so allow the package - // to go present. - if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, wzPackageId, -1, _nextPackageAfterRestart, -1)) { - // Do not allow a repair because that could put us in a perpetual restart loop. - if (BOOTSTRAPPER_REQUEST_STATE_REPAIR == *pRequestState) { - *pRequestState = BOOTSTRAPPER_REQUEST_STATE_PRESENT; - } - - ReleaseNullStr(_nextPackageAfterRestart); // no more skipping now. - } else { - // not the matching package, so skip it. - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Skipping package: %ls, after restart because it was applied before the restart.", wzPackageId); - - *pRequestState = BOOTSTRAPPER_REQUEST_STATE_NONE; - } - } else if ((_plannedAction == BOOTSTRAPPER_ACTION_INSTALL || _plannedAction == BOOTSTRAPPER_ACTION_MODIFY) && - SUCCEEDED(BalInfoFindPackageById(&_bundle.packages, wzPackageId, &pPackage))) { - BOOL f = FALSE; - if (SUCCEEDED(_engine->EvaluateCondition(pPackage->sczInstallCondition, &f)) && f) { - *pRequestState = BOOTSTRAPPER_REQUEST_STATE_PRESENT; - } - } - - return CheckCanceled() ? IDCANCEL : IDOK; - } - - virtual STDMETHODIMP_(int) OnPlanMsiFeature( - __in_z LPCWSTR wzPackageId, - __in_z LPCWSTR wzFeatureId, - __inout BOOTSTRAPPER_FEATURE_STATE* pRequestedState - ) { - LONGLONG install; - - if (wcscmp(wzFeatureId, L"AssociateFiles") == 0 || wcscmp(wzFeatureId, L"Shortcuts") == 0) { - if (SUCCEEDED(_engine->GetVariableNumeric(wzFeatureId, &install)) && install) { - *pRequestedState = BOOTSTRAPPER_FEATURE_STATE_LOCAL; - } else { - *pRequestedState = BOOTSTRAPPER_FEATURE_STATE_ABSENT; - } - } else { - *pRequestedState = BOOTSTRAPPER_FEATURE_STATE_LOCAL; - } - return CheckCanceled() ? IDCANCEL : IDNOACTION; - } - - virtual STDMETHODIMP_(void) OnPlanComplete(__in HRESULT hrStatus) { - if (SUCCEEDED(hrStatus) && _baFunction) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Running plan complete BA function"); - _baFunction->OnPlanComplete(); - } - - SetState(PYBA_STATE_PLANNED, hrStatus); - - if (SUCCEEDED(hrStatus)) { - ::PostMessageW(_hWnd, WM_PYBA_APPLY_PACKAGES, 0, 0); - } - - _startedExecution = FALSE; - _calculatedCacheProgress = 0; - _calculatedExecuteProgress = 0; - } - - - virtual STDMETHODIMP_(int) OnCachePackageBegin( - __in_z LPCWSTR wzPackageId, - __in DWORD cCachePayloads, - __in DWORD64 dw64PackageCacheSize - ) { - if (wzPackageId && *wzPackageId) { - BAL_INFO_PACKAGE* pPackage = nullptr; - HRESULT hr = BalInfoFindPackageById(&_bundle.packages, wzPackageId, &pPackage); - LPCWSTR wz = (SUCCEEDED(hr) && pPackage->sczDisplayName) ? pPackage->sczDisplayName : wzPackageId; - - ThemeSetTextControl(_theme, ID_CACHE_PROGRESS_PACKAGE_TEXT, wz); - - // If something started executing, leave it in the overall progress text. - if (!_startedExecution) { - ThemeSetTextControl(_theme, ID_OVERALL_PROGRESS_PACKAGE_TEXT, wz); - } - } - - return __super::OnCachePackageBegin(wzPackageId, cCachePayloads, dw64PackageCacheSize); - } - - - virtual STDMETHODIMP_(int) OnCacheAcquireProgress( - __in_z LPCWSTR wzPackageOrContainerId, - __in_z_opt LPCWSTR wzPayloadId, - __in DWORD64 dw64Progress, - __in DWORD64 dw64Total, - __in DWORD dwOverallPercentage - ) { - WCHAR wzProgress[5] = { }; - -#ifdef DEBUG - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "PYBA: OnCacheAcquireProgress() - container/package: %ls, payload: %ls, progress: %I64u, total: %I64u, overall progress: %u%%", wzPackageOrContainerId, wzPayloadId, dw64Progress, dw64Total, dwOverallPercentage); -#endif - - ::StringCchPrintfW(wzProgress, countof(wzProgress), L"%u%%", dwOverallPercentage); - ThemeSetTextControl(_theme, ID_CACHE_PROGRESS_TEXT, wzProgress); - - ThemeSetProgressControl(_theme, ID_CACHE_PROGRESS_BAR, dwOverallPercentage); - - _calculatedCacheProgress = dwOverallPercentage * PYBA_ACQUIRE_PERCENTAGE / 100; - ThemeSetProgressControl(_theme, ID_OVERALL_CALCULATED_PROGRESS_BAR, _calculatedCacheProgress + _calculatedExecuteProgress); - - SetTaskbarButtonProgress(_calculatedCacheProgress + _calculatedExecuteProgress); - - return __super::OnCacheAcquireProgress(wzPackageOrContainerId, wzPayloadId, dw64Progress, dw64Total, dwOverallPercentage); - } - - - virtual STDMETHODIMP_(int) OnCacheAcquireComplete( - __in_z LPCWSTR wzPackageOrContainerId, - __in_z_opt LPCWSTR wzPayloadId, - __in HRESULT hrStatus, - __in int nRecommendation - ) { - SetProgressState(hrStatus); - return __super::OnCacheAcquireComplete(wzPackageOrContainerId, wzPayloadId, hrStatus, nRecommendation); - } - - - virtual STDMETHODIMP_(int) OnCacheVerifyComplete( - __in_z LPCWSTR wzPackageId, - __in_z LPCWSTR wzPayloadId, - __in HRESULT hrStatus, - __in int nRecommendation - ) { - SetProgressState(hrStatus); - return __super::OnCacheVerifyComplete(wzPackageId, wzPayloadId, hrStatus, nRecommendation); - } - - - virtual STDMETHODIMP_(void) OnCacheComplete(__in HRESULT /*hrStatus*/) { - ThemeSetTextControl(_theme, ID_CACHE_PROGRESS_PACKAGE_TEXT, L""); - SetState(PYBA_STATE_CACHED, S_OK); // we always return success here and let OnApplyComplete() deal with the error. - } - - - virtual STDMETHODIMP_(int) OnError( - __in BOOTSTRAPPER_ERROR_TYPE errorType, - __in LPCWSTR wzPackageId, - __in DWORD dwCode, - __in_z LPCWSTR wzError, - __in DWORD dwUIHint, - __in DWORD /*cData*/, - __in_ecount_z_opt(cData) LPCWSTR* /*rgwzData*/, - __in int nRecommendation - ) { - int nResult = nRecommendation; - LPWSTR sczError = nullptr; - - if (BOOTSTRAPPER_DISPLAY_EMBEDDED == _command.display) { - HRESULT hr = _engine->SendEmbeddedError(dwCode, wzError, dwUIHint, &nResult); - if (FAILED(hr)) { - nResult = IDERROR; - } - } else if (BOOTSTRAPPER_DISPLAY_FULL == _command.display) { - // If this is an authentication failure, let the engine try to handle it for us. - if (BOOTSTRAPPER_ERROR_TYPE_HTTP_AUTH_SERVER == errorType || BOOTSTRAPPER_ERROR_TYPE_HTTP_AUTH_PROXY == errorType) { - nResult = IDTRYAGAIN; - } else // show a generic error message box. - { - BalRetryErrorOccurred(wzPackageId, dwCode); - - if (!_showingInternalUIThisPackage) { - // If no error message was provided, use the error code to try and get an error message. - if (!wzError || !*wzError || BOOTSTRAPPER_ERROR_TYPE_WINDOWS_INSTALLER != errorType) { - HRESULT hr = StrAllocFromError(&sczError, dwCode, nullptr); - if (FAILED(hr) || !sczError || !*sczError) { - StrAllocFormatted(&sczError, L"0x%x", dwCode); - } - } - - nResult = ::MessageBoxW(_hWnd, sczError ? sczError : wzError, _theme->sczCaption, dwUIHint); - } - } - - SetProgressState(HRESULT_FROM_WIN32(dwCode)); - } else { - // just take note of the error code and let things continue. - BalRetryErrorOccurred(wzPackageId, dwCode); - } - - ReleaseStr(sczError); - return nResult; - } - - - virtual STDMETHODIMP_(int) OnExecuteMsiMessage( - __in_z LPCWSTR wzPackageId, - __in INSTALLMESSAGE mt, - __in UINT uiFlags, - __in_z LPCWSTR wzMessage, - __in DWORD cData, - __in_ecount_z_opt(cData) LPCWSTR* rgwzData, - __in int nRecommendation - ) { -#ifdef DEBUG - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "PYBA: OnExecuteMsiMessage() - package: %ls, message: %ls", wzPackageId, wzMessage); -#endif - if (BOOTSTRAPPER_DISPLAY_FULL == _command.display && (INSTALLMESSAGE_WARNING == mt || INSTALLMESSAGE_USER == mt)) { - int nResult = ::MessageBoxW(_hWnd, wzMessage, _theme->sczCaption, uiFlags); - return nResult; - } - - if (INSTALLMESSAGE_ACTIONSTART == mt) { - ThemeSetTextControl(_theme, ID_EXECUTE_PROGRESS_ACTIONDATA_TEXT, wzMessage); - } - - return __super::OnExecuteMsiMessage(wzPackageId, mt, uiFlags, wzMessage, cData, rgwzData, nRecommendation); - } - - - virtual STDMETHODIMP_(int) OnProgress(__in DWORD dwProgressPercentage, __in DWORD dwOverallProgressPercentage) { - WCHAR wzProgress[5] = { }; - -#ifdef DEBUG - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "PYBA: OnProgress() - progress: %u%%, overall progress: %u%%", dwProgressPercentage, dwOverallProgressPercentage); -#endif - - ::StringCchPrintfW(wzProgress, countof(wzProgress), L"%u%%", dwOverallProgressPercentage); - ThemeSetTextControl(_theme, ID_OVERALL_PROGRESS_TEXT, wzProgress); - - ThemeSetProgressControl(_theme, ID_OVERALL_PROGRESS_BAR, dwOverallProgressPercentage); - SetTaskbarButtonProgress(dwOverallProgressPercentage); - - return __super::OnProgress(dwProgressPercentage, dwOverallProgressPercentage); - } - - - virtual STDMETHODIMP_(int) OnExecutePackageBegin(__in_z LPCWSTR wzPackageId, __in BOOL fExecute) { - LPWSTR sczFormattedString = nullptr; - - _startedExecution = TRUE; - - if (wzPackageId && *wzPackageId) { - BAL_INFO_PACKAGE* pPackage = nullptr; - BalInfoFindPackageById(&_bundle.packages, wzPackageId, &pPackage); - - LPCWSTR wz = wzPackageId; - if (pPackage) { - LOC_STRING* pLocString = nullptr; - - switch (pPackage->type) { - case BAL_INFO_PACKAGE_TYPE_BUNDLE_ADDON: - LocGetString(_wixLoc, L"#(loc.ExecuteAddonRelatedBundleMessage)", &pLocString); - break; - - case BAL_INFO_PACKAGE_TYPE_BUNDLE_PATCH: - LocGetString(_wixLoc, L"#(loc.ExecutePatchRelatedBundleMessage)", &pLocString); - break; - - case BAL_INFO_PACKAGE_TYPE_BUNDLE_UPGRADE: - LocGetString(_wixLoc, L"#(loc.ExecuteUpgradeRelatedBundleMessage)", &pLocString); - break; - } - - if (pLocString) { - // If the wix developer is showing a hidden variable in the UI, then obviously they don't care about keeping it safe - // so don't go down the rabbit hole of making sure that this is securely freed. - BalFormatString(pLocString->wzText, &sczFormattedString); - } - - wz = sczFormattedString ? sczFormattedString : pPackage->sczDisplayName ? pPackage->sczDisplayName : wzPackageId; - } - - _showingInternalUIThisPackage = pPackage && pPackage->fDisplayInternalUI; - - ThemeSetTextControl(_theme, ID_EXECUTE_PROGRESS_PACKAGE_TEXT, wz); - ThemeSetTextControl(_theme, ID_OVERALL_PROGRESS_PACKAGE_TEXT, wz); - } else { - _showingInternalUIThisPackage = FALSE; - } - - ReleaseStr(sczFormattedString); - return __super::OnExecutePackageBegin(wzPackageId, fExecute); - } - - - virtual int __stdcall OnExecuteProgress( - __in_z LPCWSTR wzPackageId, - __in DWORD dwProgressPercentage, - __in DWORD dwOverallProgressPercentage - ) { - WCHAR wzProgress[8] = { }; - -#ifdef DEBUG - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "PYBA: OnExecuteProgress() - package: %ls, progress: %u%%, overall progress: %u%%", wzPackageId, dwProgressPercentage, dwOverallProgressPercentage); -#endif - - ::StringCchPrintfW(wzProgress, countof(wzProgress), L"%u%%", dwOverallProgressPercentage); - ThemeSetTextControl(_theme, ID_EXECUTE_PROGRESS_TEXT, wzProgress); - - ThemeSetProgressControl(_theme, ID_EXECUTE_PROGRESS_BAR, dwOverallProgressPercentage); - - _calculatedExecuteProgress = dwOverallProgressPercentage * (100 - PYBA_ACQUIRE_PERCENTAGE) / 100; - ThemeSetProgressControl(_theme, ID_OVERALL_CALCULATED_PROGRESS_BAR, _calculatedCacheProgress + _calculatedExecuteProgress); - - SetTaskbarButtonProgress(_calculatedCacheProgress + _calculatedExecuteProgress); - - return __super::OnExecuteProgress(wzPackageId, dwProgressPercentage, dwOverallProgressPercentage); - } - - - virtual STDMETHODIMP_(int) OnExecutePackageComplete( - __in_z LPCWSTR wzPackageId, - __in HRESULT hrExitCode, - __in BOOTSTRAPPER_APPLY_RESTART restart, - __in int nRecommendation - ) { - SetProgressState(hrExitCode); - - if (_wcsnicmp(wzPackageId, L"path_", 5) == 0 && SUCCEEDED(hrExitCode)) { - SendMessageTimeoutW( - HWND_BROADCAST, - WM_SETTINGCHANGE, - 0, - reinterpret_cast(L"Environment"), - SMTO_ABORTIFHUNG, - 1000, - nullptr - ); - } - - int nResult = __super::OnExecutePackageComplete(wzPackageId, hrExitCode, restart, nRecommendation); - - return nResult; - } - - - virtual STDMETHODIMP_(void) OnExecuteComplete(__in HRESULT hrStatus) { - ThemeSetTextControl(_theme, ID_EXECUTE_PROGRESS_PACKAGE_TEXT, L""); - ThemeSetTextControl(_theme, ID_EXECUTE_PROGRESS_ACTIONDATA_TEXT, L""); - ThemeSetTextControl(_theme, ID_OVERALL_PROGRESS_PACKAGE_TEXT, L""); - ThemeControlEnable(_theme, ID_PROGRESS_CANCEL_BUTTON, FALSE); // no more cancel. - - SetState(PYBA_STATE_EXECUTED, S_OK); // we always return success here and let OnApplyComplete() deal with the error. - SetProgressState(hrStatus); - } - - - virtual STDMETHODIMP_(int) OnResolveSource( - __in_z LPCWSTR wzPackageOrContainerId, - __in_z_opt LPCWSTR wzPayloadId, - __in_z LPCWSTR wzLocalSource, - __in_z_opt LPCWSTR wzDownloadSource - ) { - int nResult = IDERROR; // assume we won't resolve source and that is unexpected. - - if (BOOTSTRAPPER_DISPLAY_FULL == _command.display) { - if (wzDownloadSource) { - nResult = IDDOWNLOAD; - } else { - // prompt to change the source location. - OPENFILENAMEW ofn = { }; - WCHAR wzFile[MAX_PATH] = { }; - - ::StringCchCopyW(wzFile, countof(wzFile), wzLocalSource); - - ofn.lStructSize = sizeof(ofn); - ofn.hwndOwner = _hWnd; - ofn.lpstrFile = wzFile; - ofn.nMaxFile = countof(wzFile); - ofn.lpstrFilter = L"All Files\0*.*\0"; - ofn.nFilterIndex = 1; - ofn.Flags = OFN_PATHMUSTEXIST | OFN_FILEMUSTEXIST; - ofn.lpstrTitle = _theme->sczCaption; - - if (::GetOpenFileNameW(&ofn)) { - HRESULT hr = _engine->SetLocalSource(wzPackageOrContainerId, wzPayloadId, ofn.lpstrFile); - nResult = SUCCEEDED(hr) ? IDRETRY : IDERROR; - } else { - nResult = IDCANCEL; - } - } - } else if (wzDownloadSource) { - // If doing a non-interactive install and download source is available, let's try downloading the package silently - nResult = IDDOWNLOAD; - } - // else there's nothing more we can do in non-interactive mode - - return CheckCanceled() ? IDCANCEL : nResult; - } - - - virtual STDMETHODIMP_(int) OnApplyComplete(__in HRESULT hrStatus, __in BOOTSTRAPPER_APPLY_RESTART restart) { - _restartResult = restart; // remember the restart result so we return the correct error code no matter what the user chooses to do in the UI. - - // If a restart was encountered and we are not suppressing restarts, then restart is required. - _restartRequired = (BOOTSTRAPPER_APPLY_RESTART_NONE != restart && BOOTSTRAPPER_RESTART_NEVER < _command.restart); - // If a restart is required and we're not displaying a UI or we are not supposed to prompt for restart then allow the restart. - _allowRestart = _restartRequired && (BOOTSTRAPPER_DISPLAY_FULL > _command.display || BOOTSTRAPPER_RESTART_PROMPT < _command.restart); - - // If we are showing UI, wait a beat before moving to the final screen. - if (BOOTSTRAPPER_DISPLAY_NONE < _command.display) { - ::Sleep(250); - } - - SetState(PYBA_STATE_APPLIED, hrStatus); - SetTaskbarButtonProgress(100); // show full progress bar, green, yellow, or red - - return IDNOACTION; - } - - virtual STDMETHODIMP_(void) OnLaunchApprovedExeComplete(__in HRESULT hrStatus, __in DWORD /*processId*/) { - } - - -private: - // - // UiThreadProc - entrypoint for UI thread. - // - static DWORD WINAPI UiThreadProc(__in LPVOID pvContext) { - HRESULT hr = S_OK; - PythonBootstrapperApplication* pThis = (PythonBootstrapperApplication*)pvContext; - BOOL comInitialized = FALSE; - BOOL ret = FALSE; - MSG msg = { }; - - // Initialize COM and theme. - hr = ::CoInitialize(nullptr); - BalExitOnFailure(hr, "Failed to initialize COM."); - comInitialized = TRUE; - - hr = ThemeInitialize(pThis->_hModule); - BalExitOnFailure(hr, "Failed to initialize theme manager."); - - hr = pThis->InitializeData(); - BalExitOnFailure(hr, "Failed to initialize data in bootstrapper application."); - - // Create main window. - pThis->InitializeTaskbarButton(); - hr = pThis->CreateMainWindow(); - BalExitOnFailure(hr, "Failed to create main window."); - - pThis->ValidateOperatingSystem(); - - if (FAILED(pThis->_hrFinal)) { - pThis->SetState(PYBA_STATE_FAILED, hr); - ::PostMessageW(pThis->_hWnd, WM_PYBA_SHOW_FAILURE, 0, 0); - } else { - // Okay, we're ready for packages now. - pThis->SetState(PYBA_STATE_INITIALIZED, hr); - ::PostMessageW(pThis->_hWnd, BOOTSTRAPPER_ACTION_HELP == pThis->_command.action ? WM_PYBA_SHOW_HELP : WM_PYBA_DETECT_PACKAGES, 0, 0); - } - - // message pump - while (0 != (ret = ::GetMessageW(&msg, nullptr, 0, 0))) { - if (-1 == ret) { - hr = E_UNEXPECTED; - BalExitOnFailure(hr, "Unexpected return value from message pump."); - } else if (!ThemeHandleKeyboardMessage(pThis->_theme, msg.hwnd, &msg)) { - ::TranslateMessage(&msg); - ::DispatchMessageW(&msg); - } - } - - // Succeeded thus far, check to see if anything went wrong while actually - // executing changes. - if (FAILED(pThis->_hrFinal)) { - hr = pThis->_hrFinal; - } else if (pThis->CheckCanceled()) { - hr = HRESULT_FROM_WIN32(ERROR_INSTALL_USEREXIT); - } - - LExit: - // destroy main window - pThis->DestroyMainWindow(); - - // initiate engine shutdown - DWORD dwQuit = HRESULT_CODE(hr); - if (BOOTSTRAPPER_APPLY_RESTART_INITIATED == pThis->_restartResult) { - dwQuit = ERROR_SUCCESS_REBOOT_INITIATED; - } else if (BOOTSTRAPPER_APPLY_RESTART_REQUIRED == pThis->_restartResult) { - dwQuit = ERROR_SUCCESS_REBOOT_REQUIRED; - } - pThis->_engine->Quit(dwQuit); - - ReleaseTheme(pThis->_theme); - ThemeUninitialize(); - - // uninitialize COM - if (comInitialized) { - ::CoUninitialize(); - } - - return hr; - } - - // - // ParseVariablesFromUnattendXml - reads options from unattend.xml if it - // exists - // - HRESULT ParseVariablesFromUnattendXml() { - HRESULT hr = S_OK; - LPWSTR sczUnattendXmlPath = nullptr; - IXMLDOMDocument *pixdUnattend = nullptr; - IXMLDOMNodeList *pNodes = nullptr; - IXMLDOMNode *pNode = nullptr; - long cNodes; - DWORD dwAttr; - LPWSTR scz = nullptr; - BOOL bValue; - int iValue; - BOOL tryConvert; - BSTR bstrValue = nullptr; - - hr = BalFormatString(L"[WixBundleOriginalSourceFolder]unattend.xml", &sczUnattendXmlPath); - BalExitOnFailure(hr, "Failed to calculate path to unattend.xml"); - - if (!FileExistsEx(sczUnattendXmlPath, &dwAttr)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_VERBOSE, "Did not find %ls", sczUnattendXmlPath); - hr = S_FALSE; - goto LExit; - } - - hr = XmlLoadDocumentFromFile(sczUnattendXmlPath, &pixdUnattend); - BalExitOnFailure1(hr, "Failed to read %ls", sczUnattendXmlPath); - - // get the list of variables users have overridden - hr = XmlSelectNodes(pixdUnattend, L"/Options/Option", &pNodes); - if (S_FALSE == hr) { - ExitFunction1(hr = S_OK); - } - BalExitOnFailure(hr, "Failed to select option nodes."); - - hr = pNodes->get_length((long*)&cNodes); - BalExitOnFailure(hr, "Failed to get option node count."); - - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Reading settings from %ls", sczUnattendXmlPath); - - for (DWORD i = 0; i < cNodes; ++i) { - hr = XmlNextElement(pNodes, &pNode, nullptr); - BalExitOnFailure(hr, "Failed to get next node."); - - // @Name - hr = XmlGetAttributeEx(pNode, L"Name", &scz); - BalExitOnFailure(hr, "Failed to get @Name."); - - tryConvert = TRUE; - hr = XmlGetAttribute(pNode, L"Value", &bstrValue); - if (FAILED(hr) || !bstrValue || !*bstrValue) { - hr = XmlGetText(pNode, &bstrValue); - tryConvert = FALSE; - } - BalExitOnFailure(hr, "Failed to get @Value."); - - if (tryConvert && - CSTR_EQUAL == ::CompareStringW(LOCALE_INVARIANT, NORM_IGNORECASE, bstrValue, -1, L"yes", -1)) { - _engine->SetVariableNumeric(scz, 1); - } else if (tryConvert && - CSTR_EQUAL == ::CompareStringW(LOCALE_INVARIANT, NORM_IGNORECASE, bstrValue, -1, L"no", -1)) { - _engine->SetVariableNumeric(scz, 0); - } else if (tryConvert && ::StrToIntExW(bstrValue, STIF_DEFAULT, &iValue)) { - _engine->SetVariableNumeric(scz, iValue); - } else { - _engine->SetVariableString(scz, bstrValue); - } - - ReleaseNullBSTR(bstrValue); - ReleaseNullStr(scz); - ReleaseNullObject(pNode); - } - - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Finished reading from %ls", sczUnattendXmlPath); - - LExit: - ReleaseObject(pNode); - ReleaseObject(pNodes); - ReleaseObject(pixdUnattend); - ReleaseStr(sczUnattendXmlPath); - - return hr; - } - - - // - // InitializeData - initializes all the package information. - // - HRESULT InitializeData() { - HRESULT hr = S_OK; - LPWSTR sczModulePath = nullptr; - IXMLDOMDocument *pixdManifest = nullptr; - - hr = BalManifestLoad(_hModule, &pixdManifest); - BalExitOnFailure(hr, "Failed to load bootstrapper application manifest."); - - hr = ParseOverridableVariablesFromXml(pixdManifest); - BalExitOnFailure(hr, "Failed to read overridable variables."); - - if (_command.action == BOOTSTRAPPER_ACTION_MODIFY) { - LoadOptionalFeatureStates(_engine); - } - - hr = ParseVariablesFromUnattendXml(); - ExitOnFailure(hr, "Failed to read unattend.ini file."); - - hr = ProcessCommandLine(&_language); - ExitOnFailure(hr, "Unknown commandline parameters."); - - hr = PathRelativeToModule(&sczModulePath, nullptr, _hModule); - BalExitOnFailure(hr, "Failed to get module path."); - - hr = LoadLocalization(sczModulePath, _language); - ExitOnFailure(hr, "Failed to load localization."); - - hr = LoadTheme(sczModulePath, _language); - ExitOnFailure(hr, "Failed to load theme."); - - hr = BalInfoParseFromXml(&_bundle, pixdManifest); - BalExitOnFailure(hr, "Failed to load bundle information."); - - hr = BalConditionsParseFromXml(&_conditions, pixdManifest, _wixLoc); - BalExitOnFailure(hr, "Failed to load conditions from XML."); - - hr = LoadBootstrapperBAFunctions(); - BalExitOnFailure(hr, "Failed to load bootstrapper functions."); - - hr = UpdateUIStrings(_command.action); - BalExitOnFailure(hr, "Failed to load UI strings."); - - GetBundleFileVersion(); - // don't fail if we couldn't get the version info; best-effort only - LExit: - ReleaseObject(pixdManifest); - ReleaseStr(sczModulePath); - - return hr; - } - - - // - // ProcessCommandLine - process the provided command line arguments. - // - HRESULT ProcessCommandLine(__inout LPWSTR* psczLanguage) { - HRESULT hr = S_OK; - int argc = 0; - LPWSTR* argv = nullptr; - LPWSTR sczVariableName = nullptr; - LPWSTR sczVariableValue = nullptr; - - if (_command.wzCommandLine && *_command.wzCommandLine) { - argv = ::CommandLineToArgvW(_command.wzCommandLine, &argc); - ExitOnNullWithLastError(argv, hr, "Failed to get command line."); - - for (int i = 0; i < argc; ++i) { - if (argv[i][0] == L'-' || argv[i][0] == L'/') { - if (CSTR_EQUAL == ::CompareStringW(LOCALE_INVARIANT, NORM_IGNORECASE, &argv[i][1], -1, L"lang", -1)) { - if (i + 1 >= argc) { - hr = E_INVALIDARG; - BalExitOnFailure(hr, "Must specify a language."); - } - - ++i; - - hr = StrAllocString(psczLanguage, &argv[i][0], 0); - BalExitOnFailure(hr, "Failed to copy language."); - } else if (CSTR_EQUAL == ::CompareStringW(LOCALE_INVARIANT, NORM_IGNORECASE, &argv[i][1], -1, L"simple", -1)) { - _engine->SetVariableNumeric(L"SimpleInstall", 1); - } - } else if (_overridableVariables) { - int value; - const wchar_t* pwc = wcschr(argv[i], L'='); - if (pwc) { - hr = StrAllocString(&sczVariableName, argv[i], pwc - argv[i]); - BalExitOnFailure(hr, "Failed to copy variable name."); - - hr = DictKeyExists(_overridableVariables, sczVariableName); - if (E_NOTFOUND == hr) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Ignoring attempt to set non-overridable variable: '%ls'.", sczVariableName); - hr = S_OK; - continue; - } - ExitOnFailure(hr, "Failed to check the dictionary of overridable variables."); - - hr = StrAllocString(&sczVariableValue, ++pwc, 0); - BalExitOnFailure(hr, "Failed to copy variable value."); - - if (::StrToIntEx(sczVariableValue, STIF_DEFAULT, &value)) { - hr = _engine->SetVariableNumeric(sczVariableName, value); - } else { - hr = _engine->SetVariableString(sczVariableName, sczVariableValue); - } - BalExitOnFailure(hr, "Failed to set variable."); - } else { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Ignoring unknown argument: %ls", argv[i]); - } - } - } - } - - LExit: - if (argv) { - ::LocalFree(argv); - } - - ReleaseStr(sczVariableName); - ReleaseStr(sczVariableValue); - - return hr; - } - - HRESULT LoadLocalization(__in_z LPCWSTR wzModulePath, __in_z_opt LPCWSTR wzLanguage) { - HRESULT hr = S_OK; - LPWSTR sczLocPath = nullptr; - LPCWSTR wzLocFileName = L"Default.wxl"; - - hr = LocProbeForFile(wzModulePath, wzLocFileName, wzLanguage, &sczLocPath); - BalExitOnFailure2(hr, "Failed to probe for loc file: %ls in path: %ls", wzLocFileName, wzModulePath); - - hr = LocLoadFromFile(sczLocPath, &_wixLoc); - BalExitOnFailure1(hr, "Failed to load loc file from path: %ls", sczLocPath); - - if (WIX_LOCALIZATION_LANGUAGE_NOT_SET != _wixLoc->dwLangId) { - ::SetThreadLocale(_wixLoc->dwLangId); - } - - hr = StrAllocString(&_confirmCloseMessage, L"#(loc.ConfirmCancelMessage)", 0); - ExitOnFailure(hr, "Failed to initialize confirm message loc identifier."); - - hr = LocLocalizeString(_wixLoc, &_confirmCloseMessage); - BalExitOnFailure1(hr, "Failed to localize confirm close message: %ls", _confirmCloseMessage); - - LExit: - ReleaseStr(sczLocPath); - - return hr; - } - - - HRESULT LoadTheme(__in_z LPCWSTR wzModulePath, __in_z_opt LPCWSTR wzLanguage) { - HRESULT hr = S_OK; - LPWSTR sczThemePath = nullptr; - LPCWSTR wzThemeFileName = L"Default.thm"; - LPWSTR sczCaption = nullptr; - - hr = LocProbeForFile(wzModulePath, wzThemeFileName, wzLanguage, &sczThemePath); - BalExitOnFailure2(hr, "Failed to probe for theme file: %ls in path: %ls", wzThemeFileName, wzModulePath); - - hr = ThemeLoadFromFile(sczThemePath, &_theme); - BalExitOnFailure1(hr, "Failed to load theme from path: %ls", sczThemePath); - - hr = ThemeLocalize(_theme, _wixLoc); - BalExitOnFailure1(hr, "Failed to localize theme: %ls", sczThemePath); - - // Update the caption if there are any formatted strings in it. - // If the wix developer is showing a hidden variable in the UI, then - // obviously they don't care about keeping it safe so don't go down the - // rabbit hole of making sure that this is securely freed. - hr = BalFormatString(_theme->sczCaption, &sczCaption); - if (SUCCEEDED(hr)) { - ThemeUpdateCaption(_theme, sczCaption); - } - - LExit: - ReleaseStr(sczCaption); - ReleaseStr(sczThemePath); - - return hr; - } - - - HRESULT ParseOverridableVariablesFromXml(__in IXMLDOMDocument* pixdManifest) { - HRESULT hr = S_OK; - IXMLDOMNode* pNode = nullptr; - IXMLDOMNodeList* pNodes = nullptr; - DWORD cNodes = 0; - LPWSTR scz = nullptr; - BOOL hidden = FALSE; - - // get the list of variables users can override on the command line - hr = XmlSelectNodes(pixdManifest, L"/BootstrapperApplicationData/WixStdbaOverridableVariable", &pNodes); - if (S_FALSE == hr) { - ExitFunction1(hr = S_OK); - } - ExitOnFailure(hr, "Failed to select overridable variable nodes."); - - hr = pNodes->get_length((long*)&cNodes); - ExitOnFailure(hr, "Failed to get overridable variable node count."); - - if (cNodes) { - hr = DictCreateStringList(&_overridableVariables, 32, DICT_FLAG_NONE); - ExitOnFailure(hr, "Failed to create the string dictionary."); - - for (DWORD i = 0; i < cNodes; ++i) { - hr = XmlNextElement(pNodes, &pNode, nullptr); - ExitOnFailure(hr, "Failed to get next node."); - - // @Name - hr = XmlGetAttributeEx(pNode, L"Name", &scz); - ExitOnFailure(hr, "Failed to get @Name."); - - hr = XmlGetYesNoAttribute(pNode, L"Hidden", &hidden); - - if (!hidden) { - hr = DictAddKey(_overridableVariables, scz); - ExitOnFailure1(hr, "Failed to add \"%ls\" to the string dictionary.", scz); - } - - // prepare next iteration - ReleaseNullObject(pNode); - } - } - - LExit: - ReleaseObject(pNode); - ReleaseObject(pNodes); - ReleaseStr(scz); - return hr; - } - - - // - // Get the file version of the bootstrapper and record in bootstrapper log file - // - HRESULT GetBundleFileVersion() { - HRESULT hr = S_OK; - ULARGE_INTEGER uliVersion = { }; - LPWSTR sczCurrentPath = nullptr; - - hr = PathForCurrentProcess(&sczCurrentPath, nullptr); - BalExitOnFailure(hr, "Failed to get bundle path."); - - hr = FileVersion(sczCurrentPath, &uliVersion.HighPart, &uliVersion.LowPart); - BalExitOnFailure(hr, "Failed to get bundle file version."); - - hr = _engine->SetVariableVersion(PYBA_VARIABLE_BUNDLE_FILE_VERSION, uliVersion.QuadPart); - BalExitOnFailure(hr, "Failed to set WixBundleFileVersion variable."); - - LExit: - ReleaseStr(sczCurrentPath); - - return hr; - } - - - // - // CreateMainWindow - creates the main install window. - // - HRESULT CreateMainWindow() { - HRESULT hr = S_OK; - HICON hIcon = reinterpret_cast(_theme->hIcon); - WNDCLASSW wc = { }; - DWORD dwWindowStyle = 0; - int x = CW_USEDEFAULT; - int y = CW_USEDEFAULT; - POINT ptCursor = { }; - HMONITOR hMonitor = nullptr; - MONITORINFO mi = { }; - COLORREF fg, bg; - HBRUSH bgBrush; - - // If the theme did not provide an icon, try using the icon from the bundle engine. - if (!hIcon) { - HMODULE hBootstrapperEngine = ::GetModuleHandleW(nullptr); - if (hBootstrapperEngine) { - hIcon = ::LoadIconW(hBootstrapperEngine, MAKEINTRESOURCEW(1)); - } - } - - fg = RGB(0, 0, 0); - bg = RGB(255, 255, 255); - bgBrush = (HBRUSH)(COLOR_WINDOW+1); - if (_theme->dwFontId < _theme->cFonts) { - THEME_FONT *font = &_theme->rgFonts[_theme->dwFontId]; - fg = font->crForeground; - bg = font->crBackground; - bgBrush = font->hBackground; - RemapColor(&fg, &bg, &bgBrush); - } - - // Register the window class and create the window. - wc.lpfnWndProc = PythonBootstrapperApplication::WndProc; - wc.hInstance = _hModule; - wc.hIcon = hIcon; - wc.hCursor = ::LoadCursorW(nullptr, (LPCWSTR)IDC_ARROW); - wc.hbrBackground = bgBrush; - wc.lpszMenuName = nullptr; - wc.lpszClassName = PYBA_WINDOW_CLASS; - if (!::RegisterClassW(&wc)) { - ExitWithLastError(hr, "Failed to register window."); - } - - _registered = TRUE; - - // Calculate the window style based on the theme style and command display value. - dwWindowStyle = _theme->dwStyle; - if (BOOTSTRAPPER_DISPLAY_NONE >= _command.display) { - dwWindowStyle &= ~WS_VISIBLE; - } - - // Don't show the window if there is a splash screen (it will be made visible when the splash screen is hidden) - if (::IsWindow(_command.hwndSplashScreen)) { - dwWindowStyle &= ~WS_VISIBLE; - } - - // Center the window on the monitor with the mouse. - if (::GetCursorPos(&ptCursor)) { - hMonitor = ::MonitorFromPoint(ptCursor, MONITOR_DEFAULTTONEAREST); - if (hMonitor) { - mi.cbSize = sizeof(mi); - if (::GetMonitorInfoW(hMonitor, &mi)) { - x = mi.rcWork.left + (mi.rcWork.right - mi.rcWork.left - _theme->nWidth) / 2; - y = mi.rcWork.top + (mi.rcWork.bottom - mi.rcWork.top - _theme->nHeight) / 2; - } - } - } - - _hWnd = ::CreateWindowExW( - 0, - wc.lpszClassName, - _theme->sczCaption, - dwWindowStyle, - x, - y, - _theme->nWidth, - _theme->nHeight, - HWND_DESKTOP, - nullptr, - _hModule, - this - ); - ExitOnNullWithLastError(_hWnd, hr, "Failed to create window."); - - hr = S_OK; - - LExit: - return hr; - } - - - // - // InitializeTaskbarButton - initializes taskbar button for progress. - // - void InitializeTaskbarButton() { - HRESULT hr = S_OK; - - hr = ::CoCreateInstance(CLSID_TaskbarList, nullptr, CLSCTX_ALL, __uuidof(ITaskbarList3), reinterpret_cast(&_taskbarList)); - if (REGDB_E_CLASSNOTREG == hr) { - // not supported before Windows 7 - ExitFunction1(hr = S_OK); - } - BalExitOnFailure(hr, "Failed to create ITaskbarList3. Continuing."); - - _taskbarButtonCreatedMessage = ::RegisterWindowMessageW(L"TaskbarButtonCreated"); - BalExitOnNullWithLastError(_taskbarButtonCreatedMessage, hr, "Failed to get TaskbarButtonCreated message. Continuing."); - - LExit: - return; - } - - // - // DestroyMainWindow - clean up all the window registration. - // - void DestroyMainWindow() { - if (::IsWindow(_hWnd)) { - ::DestroyWindow(_hWnd); - _hWnd = nullptr; - _taskbarButtonOK = FALSE; - } - - if (_registered) { - ::UnregisterClassW(PYBA_WINDOW_CLASS, _hModule); - _registered = FALSE; - } - } - - - // - // WndProc - standard windows message handler. - // - static LRESULT CALLBACK WndProc( - __in HWND hWnd, - __in UINT uMsg, - __in WPARAM wParam, - __in LPARAM lParam - ) { -#pragma warning(suppress:4312) - auto pBA = reinterpret_cast(::GetWindowLongPtrW(hWnd, GWLP_USERDATA)); - - switch (uMsg) { - case WM_NCCREATE: { - LPCREATESTRUCT lpcs = reinterpret_cast(lParam); - pBA = reinterpret_cast(lpcs->lpCreateParams); -#pragma warning(suppress:4244) - ::SetWindowLongPtrW(hWnd, GWLP_USERDATA, reinterpret_cast(pBA)); - break; - } - - case WM_NCDESTROY: { - LRESULT lres = ThemeDefWindowProc(pBA ? pBA->_theme : nullptr, hWnd, uMsg, wParam, lParam); - ::SetWindowLongPtrW(hWnd, GWLP_USERDATA, 0); - return lres; - } - - case WM_CREATE: - if (!pBA->OnCreate(hWnd)) { - return -1; - } - break; - - case WM_QUERYENDSESSION: - return IDCANCEL != pBA->OnSystemShutdown(static_cast(lParam), IDCANCEL); - - case WM_CLOSE: - // If the user chose not to close, do *not* let the default window proc handle the message. - if (!pBA->OnClose()) { - return 0; - } - break; - - case WM_DESTROY: - ::PostQuitMessage(0); - break; - - case WM_PAINT: __fallthrough; - case WM_ERASEBKGND: - if (pBA && pBA->_suppressPaint) { - return TRUE; - } - break; - - case WM_PYBA_SHOW_HELP: - pBA->OnShowHelp(); - return 0; - - case WM_PYBA_DETECT_PACKAGES: - pBA->OnDetect(); - return 0; - - case WM_PYBA_PLAN_PACKAGES: - pBA->OnPlan(static_cast(lParam)); - return 0; - - case WM_PYBA_APPLY_PACKAGES: - pBA->OnApply(); - return 0; - - case WM_PYBA_CHANGE_STATE: - pBA->OnChangeState(static_cast(lParam)); - return 0; - - case WM_PYBA_SHOW_FAILURE: - pBA->OnShowFailure(); - return 0; - - case WM_COMMAND: - switch (LOWORD(wParam)) { - // Customize commands - // Success/failure commands - case ID_SUCCESS_RESTART_BUTTON: __fallthrough; - case ID_FAILURE_RESTART_BUTTON: - pBA->OnClickRestartButton(); - return 0; - - case IDCANCEL: __fallthrough; - case ID_INSTALL_CANCEL_BUTTON: __fallthrough; - case ID_CUSTOM1_CANCEL_BUTTON: __fallthrough; - case ID_CUSTOM2_CANCEL_BUTTON: __fallthrough; - case ID_MODIFY_CANCEL_BUTTON: __fallthrough; - case ID_PROGRESS_CANCEL_BUTTON: __fallthrough; - case ID_SUCCESS_CANCEL_BUTTON: __fallthrough; - case ID_FAILURE_CANCEL_BUTTON: __fallthrough; - case ID_CLOSE_BUTTON: - pBA->OnCommand(ID_CLOSE_BUTTON); - return 0; - - default: - pBA->OnCommand((CONTROL_ID)LOWORD(wParam)); - } - break; - - case WM_NOTIFY: - if (lParam) { - LPNMHDR pnmhdr = reinterpret_cast(lParam); - switch (pnmhdr->code) { - case NM_CLICK: __fallthrough; - case NM_RETURN: - switch (static_cast(pnmhdr->idFrom)) { - case ID_FAILURE_LOGFILE_LINK: - pBA->OnClickLogFileLink(); - return 1; - } - } - } - break; - - case WM_CTLCOLORSTATIC: - case WM_CTLCOLORBTN: - if (pBA) { - HBRUSH brush = nullptr; - if (pBA->SetControlColor((HWND)lParam, (HDC)wParam, &brush)) { - return (LRESULT)brush; - } - } - break; - } - - if (pBA && pBA->_taskbarList && uMsg == pBA->_taskbarButtonCreatedMessage) { - pBA->_taskbarButtonOK = TRUE; - return 0; - } - - return ThemeDefWindowProc(pBA ? pBA->_theme : nullptr, hWnd, uMsg, wParam, lParam); - } - - // - // OnCreate - finishes loading the theme. - // - BOOL OnCreate(__in HWND hWnd) { - HRESULT hr = S_OK; - - hr = ThemeLoadControls(_theme, hWnd, CONTROL_ID_NAMES, countof(CONTROL_ID_NAMES)); - BalExitOnFailure(hr, "Failed to load theme controls."); - - C_ASSERT(COUNT_PAGE == countof(PAGE_NAMES)); - C_ASSERT(countof(_pageIds) == countof(PAGE_NAMES)); - - ThemeGetPageIds(_theme, PAGE_NAMES, _pageIds, countof(_pageIds)); - - // Initialize the text on all "application" (non-page) controls. - for (DWORD i = 0; i < _theme->cControls; ++i) { - THEME_CONTROL* pControl = _theme->rgControls + i; - LPWSTR text = nullptr; - - if (!pControl->wPageId && pControl->sczText && *pControl->sczText) { - HRESULT hrFormat; - - // If the wix developer is showing a hidden variable in the UI, - // then obviously they don't care about keeping it safe so don't - // go down the rabbit hole of making sure that this is securely - // freed. - hrFormat = BalFormatString(pControl->sczText, &text); - if (SUCCEEDED(hrFormat)) { - ThemeSetTextControl(_theme, pControl->wId, text); - ReleaseStr(text); - } - } - } - - LExit: - return SUCCEEDED(hr); - } - - void RemapColor(COLORREF *fg, COLORREF *bg, HBRUSH *bgBrush) { - if (*fg == RGB(0, 0, 0)) { - *fg = GetSysColor(COLOR_WINDOWTEXT); - } else if (*fg == RGB(128, 128, 128)) { - *fg = GetSysColor(COLOR_GRAYTEXT); - } - if (*bgBrush && *bg == RGB(255, 255, 255)) { - *bg = GetSysColor(COLOR_WINDOW); - *bgBrush = GetSysColorBrush(COLOR_WINDOW); - } - } - - BOOL SetControlColor(HWND hWnd, HDC hDC, HBRUSH *brush) { - for (int i = 0; i < _theme->cControls; ++i) { - if (_theme->rgControls[i].hWnd != hWnd) { - continue; - } - - DWORD fontId = _theme->rgControls[i].dwFontId; - if (fontId > _theme->cFonts) { - fontId = 0; - } - THEME_FONT *fnt = &_theme->rgFonts[fontId]; - - COLORREF fg = fnt->crForeground, bg = fnt->crBackground; - *brush = fnt->hBackground; - RemapColor(&fg, &bg, brush); - ::SetTextColor(hDC, fg); - ::SetBkColor(hDC, bg); - - return TRUE; - } - return FALSE; - } - - // - // OnShowFailure - display the failure page. - // - void OnShowFailure() { - SetState(PYBA_STATE_FAILED, S_OK); - - // If the UI should be visible, display it now and hide the splash screen - if (BOOTSTRAPPER_DISPLAY_NONE < _command.display) { - ::ShowWindow(_theme->hwndParent, SW_SHOW); - } - - _engine->CloseSplashScreen(); - - return; - } - - - // - // OnShowHelp - display the help page. - // - void OnShowHelp() { - SetState(PYBA_STATE_HELP, S_OK); - - // If the UI should be visible, display it now and hide the splash screen - if (BOOTSTRAPPER_DISPLAY_NONE < _command.display) { - ::ShowWindow(_theme->hwndParent, SW_SHOW); - } - - _engine->CloseSplashScreen(); - - return; - } - - - // - // OnDetect - start the processing of packages. - // - void OnDetect() { - HRESULT hr = S_OK; - - if (_baFunction) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Running detect BA function"); - hr = _baFunction->OnDetect(); - BalExitOnFailure(hr, "Failed calling detect BA function."); - } - - SetState(PYBA_STATE_DETECTING, hr); - - // If the UI should be visible, display it now and hide the splash screen - if (BOOTSTRAPPER_DISPLAY_NONE < _command.display) { - ::ShowWindow(_theme->hwndParent, SW_SHOW); - } - - _engine->CloseSplashScreen(); - - // Tell the core we're ready for the packages to be processed now. - hr = _engine->Detect(); - BalExitOnFailure(hr, "Failed to start detecting chain."); - - LExit: - if (FAILED(hr)) { - SetState(PYBA_STATE_DETECTING, hr); - } - - return; - } - - HRESULT UpdateUIStrings(__in BOOTSTRAPPER_ACTION action) { - HRESULT hr = S_OK; - LPCWSTR likeInstalling = nullptr; - LPCWSTR likeInstallation = nullptr; - switch (action) { - case BOOTSTRAPPER_ACTION_INSTALL: - likeInstalling = L"Installing"; - likeInstallation = L"Installation"; - break; - case BOOTSTRAPPER_ACTION_MODIFY: - // For modify, we actually want to pass INSTALL - action = BOOTSTRAPPER_ACTION_INSTALL; - likeInstalling = L"Modifying"; - likeInstallation = L"Modification"; - break; - case BOOTSTRAPPER_ACTION_REPAIR: - likeInstalling = L"Repairing"; - likeInstallation = L"Repair"; - break; - case BOOTSTRAPPER_ACTION_UNINSTALL: - likeInstalling = L"Uninstalling"; - likeInstallation = L"Uninstallation"; - break; - } - - if (likeInstalling) { - LPWSTR locName = nullptr; - LOC_STRING *locText = nullptr; - hr = StrAllocFormatted(&locName, L"#(loc.%ls)", likeInstalling); - if (SUCCEEDED(hr)) { - hr = LocGetString(_wixLoc, locName, &locText); - ReleaseStr(locName); - } - _engine->SetVariableString( - L"ActionLikeInstalling", - SUCCEEDED(hr) && locText ? locText->wzText : likeInstalling - ); - } - - if (likeInstallation) { - LPWSTR locName = nullptr; - LOC_STRING *locText = nullptr; - hr = StrAllocFormatted(&locName, L"#(loc.%ls)", likeInstallation); - if (SUCCEEDED(hr)) { - hr = LocGetString(_wixLoc, locName, &locText); - ReleaseStr(locName); - } - _engine->SetVariableString( - L"ActionLikeInstallation", - SUCCEEDED(hr) && locText ? locText->wzText : likeInstallation - ); - } - return hr; - } - - // - // OnPlan - plan the detected changes. - // - void OnPlan(__in BOOTSTRAPPER_ACTION action) { - HRESULT hr = S_OK; - - _plannedAction = action; - - hr = UpdateUIStrings(action); - BalExitOnFailure(hr, "Failed to update strings"); - - // If we are going to apply a downgrade, bail. - if (_downgradingOtherVersion && BOOTSTRAPPER_ACTION_UNINSTALL < action) { - if (_suppressDowngradeFailure) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "A newer version of this product is installed but downgrade failure has been suppressed; continuing..."); - } else { - hr = HRESULT_FROM_WIN32(ERROR_PRODUCT_VERSION); - BalExitOnFailure(hr, "Cannot install a product when a newer version is installed."); - } - } - - SetState(PYBA_STATE_PLANNING, hr); - - if (_baFunction) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Running plan BA function"); - _baFunction->OnPlan(); - } - - hr = _engine->Plan(action); - BalExitOnFailure(hr, "Failed to start planning packages."); - - LExit: - if (FAILED(hr)) { - SetState(PYBA_STATE_PLANNING, hr); - } - - return; - } - - - // - // OnApply - apply the packages. - // - void OnApply() { - HRESULT hr = S_OK; - - SetState(PYBA_STATE_APPLYING, hr); - SetProgressState(hr); - SetTaskbarButtonProgress(0); - - hr = _engine->Apply(_hWnd); - BalExitOnFailure(hr, "Failed to start applying packages."); - - ThemeControlEnable(_theme, ID_PROGRESS_CANCEL_BUTTON, TRUE); // ensure the cancel button is enabled before starting. - - LExit: - if (FAILED(hr)) { - SetState(PYBA_STATE_APPLYING, hr); - } - - return; - } - - - // - // OnChangeState - change state. - // - void OnChangeState(__in PYBA_STATE state) { - LPWSTR unformattedText = nullptr; - - _state = state; - - // If our install is at the end (success or failure) and we're not showing full UI - // then exit (prompt for restart if required). - if ((PYBA_STATE_APPLIED <= _state && BOOTSTRAPPER_DISPLAY_FULL > _command.display)) { - // If a restart was required but we were not automatically allowed to - // accept the reboot then do the prompt. - if (_restartRequired && !_allowRestart) { - StrAllocFromError(&unformattedText, HRESULT_FROM_WIN32(ERROR_SUCCESS_REBOOT_REQUIRED), nullptr); - - _allowRestart = IDOK == ::MessageBoxW( - _hWnd, - unformattedText ? unformattedText : L"The requested operation is successful. Changes will not be effective until the system is rebooted.", - _theme->sczCaption, - MB_ICONEXCLAMATION | MB_OKCANCEL - ); - } - - // Quietly exit. - ::PostMessageW(_hWnd, WM_CLOSE, 0, 0); - } else { // try to change the pages. - DWORD newPageId = 0; - DeterminePageId(_state, &newPageId); - - if (_visiblePageId != newPageId) { - ShowPage(newPageId); - } - } - - ReleaseStr(unformattedText); - } - - // - // Called before showing a page to handle all controls. - // - void ProcessPageControls(THEME_PAGE *pPage) { - if (!pPage) { - return; - } - - for (DWORD i = 0; i < pPage->cControlIndices; ++i) { - THEME_CONTROL* pControl = _theme->rgControls + pPage->rgdwControlIndices[i]; - BOOL enableControl = TRUE; - - // If this is a named control, try to set its default state. - if (pControl->sczName && *pControl->sczName) { - // If this is a checkable control, try to set its default state - // to the state of a matching named Burn variable. - if (IsCheckable(pControl)) { - LONGLONG llValue = 0; - HRESULT hr = BalGetNumericVariable(pControl->sczName, &llValue); - - // If the control value isn't set then disable it. - if (!SUCCEEDED(hr)) { - enableControl = FALSE; - } else { - ThemeSendControlMessage( - _theme, - pControl->wId, - BM_SETCHECK, - SUCCEEDED(hr) && llValue ? BST_CHECKED : BST_UNCHECKED, - 0 - ); - } - } - - // Hide or disable controls based on the control name with 'State' appended - LPWSTR controlName = nullptr; - HRESULT hr = StrAllocFormatted(&controlName, L"%lsState", pControl->sczName); - if (SUCCEEDED(hr)) { - LPWSTR controlState = nullptr; - hr = BalGetStringVariable(controlName, &controlState); - if (SUCCEEDED(hr) && controlState && *controlState) { - if (controlState[0] == '[') { - LPWSTR formatted = nullptr; - if (SUCCEEDED(BalFormatString(controlState, &formatted))) { - StrFree(controlState); - controlState = formatted; - } - } - - if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, controlState, -1, L"disable", -1)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Disable control %ls", pControl->sczName); - enableControl = FALSE; - } else if (CSTR_EQUAL == ::CompareStringW(LOCALE_NEUTRAL, 0, controlState, -1, L"hide", -1)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Hide control %ls", pControl->sczName); - // TODO: This doesn't work - ThemeShowControl(_theme, pControl->wId, SW_HIDE); - } else { - // An explicit state can override the lack of a - // backing variable. - enableControl = TRUE; - } - } - StrFree(controlState); - } - StrFree(controlName); - controlName = nullptr; - - - // If a command link has a note, then add it. - if ((pControl->dwStyle & BS_TYPEMASK) == BS_COMMANDLINK || - (pControl->dwStyle & BS_TYPEMASK) == BS_DEFCOMMANDLINK) { - hr = StrAllocFormatted(&controlName, L"#(loc.%lsNote)", pControl->sczName); - if (SUCCEEDED(hr)) { - LOC_STRING *locText = nullptr; - hr = LocGetString(_wixLoc, controlName, &locText); - if (SUCCEEDED(hr) && locText && locText->wzText && locText->wzText[0]) { - LPWSTR text = nullptr; - hr = BalFormatString(locText->wzText, &text); - if (SUCCEEDED(hr) && text && text[0]) { - ThemeSendControlMessage(_theme, pControl->wId, BCM_SETNOTE, 0, (LPARAM)text); - ReleaseStr(text); - text = nullptr; - } - } - ReleaseStr(controlName); - controlName = nullptr; - } - hr = S_OK; - } - } - - ThemeControlEnable(_theme, pControl->wId, enableControl); - - // Format the text in each of the new page's controls - if (pControl->sczText && *pControl->sczText) { - // If the wix developer is showing a hidden variable - // in the UI, then obviously they don't care about - // keeping it safe so don't go down the rabbit hole - // of making sure that this is securely freed. - LPWSTR text = nullptr; - HRESULT hr = BalFormatString(pControl->sczText, &text); - if (SUCCEEDED(hr)) { - ThemeSetTextControl(_theme, pControl->wId, text); - } - } - } - } - - // - // OnClose - called when the window is trying to be closed. - // - BOOL OnClose() { - BOOL close = FALSE; - - // If we've already succeeded or failed or showing the help page, just close (prompts are annoying if the bootstrapper is done). - if (PYBA_STATE_APPLIED <= _state || PYBA_STATE_HELP == _state) { - close = TRUE; - } else { - // prompt the user or force the cancel if there is no UI. - close = PromptCancel( - _hWnd, - BOOTSTRAPPER_DISPLAY_FULL != _command.display, - _confirmCloseMessage ? _confirmCloseMessage : L"Are you sure you want to cancel?", - _theme->sczCaption - ); - } - - // If we're doing progress then we never close, we just cancel to let rollback occur. - if (PYBA_STATE_APPLYING <= _state && PYBA_STATE_APPLIED > _state) { - // If we canceled disable cancel button since clicking it again is silly. - if (close) { - ThemeControlEnable(_theme, ID_PROGRESS_CANCEL_BUTTON, FALSE); - } - - close = FALSE; - } - - return close; - } - - // - // OnClickCloseButton - close the application. - // - void OnClickCloseButton() { - ::SendMessageW(_hWnd, WM_CLOSE, 0, 0); - } - - - - // - // OnClickRestartButton - allows the restart and closes the app. - // - void OnClickRestartButton() { - AssertSz(_restartRequired, "Restart must be requested to be able to click on the restart button."); - - _allowRestart = TRUE; - ::SendMessageW(_hWnd, WM_CLOSE, 0, 0); - - return; - } - - - // - // OnClickLogFileLink - show the log file. - // - void OnClickLogFileLink() { - HRESULT hr = S_OK; - LPWSTR sczLogFile = nullptr; - - hr = BalGetStringVariable(_bundle.sczLogVariable, &sczLogFile); - BalExitOnFailure1(hr, "Failed to get log file variable '%ls'.", _bundle.sczLogVariable); - - hr = ShelExec(L"notepad.exe", sczLogFile, L"open", nullptr, SW_SHOWDEFAULT, _hWnd, nullptr); - BalExitOnFailure1(hr, "Failed to open log file target: %ls", sczLogFile); - - LExit: - ReleaseStr(sczLogFile); - - return; - } - - - // - // SetState - // - void SetState(__in PYBA_STATE state, __in HRESULT hrStatus) { - if (FAILED(hrStatus)) { - _hrFinal = hrStatus; - } - - if (FAILED(_hrFinal)) { - state = PYBA_STATE_FAILED; - } - - if (_state != state) { - ::PostMessageW(_hWnd, WM_PYBA_CHANGE_STATE, 0, state); - } - } - - // - // GoToPage - // - void GoToPage(__in PAGE page) { - _installPage = page; - ::PostMessageW(_hWnd, WM_PYBA_CHANGE_STATE, 0, _state); - } - - void DeterminePageId(__in PYBA_STATE state, __out DWORD* pdwPageId) { - LONGLONG simple; - - if (BOOTSTRAPPER_DISPLAY_PASSIVE == _command.display) { - switch (state) { - case PYBA_STATE_INITIALIZED: - *pdwPageId = BOOTSTRAPPER_ACTION_HELP == _command.action - ? _pageIds[PAGE_HELP] - : _pageIds[PAGE_LOADING]; - break; - - case PYBA_STATE_HELP: - *pdwPageId = _pageIds[PAGE_HELP]; - break; - - case PYBA_STATE_DETECTING: - *pdwPageId = _pageIds[PAGE_LOADING] - ? _pageIds[PAGE_LOADING] - : _pageIds[PAGE_PROGRESS_PASSIVE] - ? _pageIds[PAGE_PROGRESS_PASSIVE] - : _pageIds[PAGE_PROGRESS]; - break; - - case PYBA_STATE_DETECTED: __fallthrough; - case PYBA_STATE_PLANNING: __fallthrough; - case PYBA_STATE_PLANNED: __fallthrough; - case PYBA_STATE_APPLYING: __fallthrough; - case PYBA_STATE_CACHING: __fallthrough; - case PYBA_STATE_CACHED: __fallthrough; - case PYBA_STATE_EXECUTING: __fallthrough; - case PYBA_STATE_EXECUTED: - *pdwPageId = _pageIds[PAGE_PROGRESS_PASSIVE] - ? _pageIds[PAGE_PROGRESS_PASSIVE] - : _pageIds[PAGE_PROGRESS]; - break; - - default: - *pdwPageId = 0; - break; - } - } else if (BOOTSTRAPPER_DISPLAY_FULL == _command.display) { - switch (state) { - case PYBA_STATE_INITIALIZING: - *pdwPageId = 0; - break; - - case PYBA_STATE_INITIALIZED: - *pdwPageId = BOOTSTRAPPER_ACTION_HELP == _command.action - ? _pageIds[PAGE_HELP] - : _pageIds[PAGE_LOADING]; - break; - - case PYBA_STATE_HELP: - *pdwPageId = _pageIds[PAGE_HELP]; - break; - - case PYBA_STATE_DETECTING: - *pdwPageId = _pageIds[PAGE_LOADING]; - break; - - case PYBA_STATE_DETECTED: - if (_installPage == PAGE_LOADING) { - switch (_command.action) { - case BOOTSTRAPPER_ACTION_INSTALL: - if (_upgrading) { - _installPage = PAGE_UPGRADE; - } else if (SUCCEEDED(BalGetNumericVariable(L"SimpleInstall", &simple)) && simple) { - _installPage = PAGE_SIMPLE_INSTALL; - } else { - _installPage = PAGE_INSTALL; - } - break; - - case BOOTSTRAPPER_ACTION_MODIFY: __fallthrough; - case BOOTSTRAPPER_ACTION_REPAIR: __fallthrough; - case BOOTSTRAPPER_ACTION_UNINSTALL: - _installPage = PAGE_MODIFY; - break; - } - } - *pdwPageId = _pageIds[_installPage]; - break; - - case PYBA_STATE_PLANNING: __fallthrough; - case PYBA_STATE_PLANNED: __fallthrough; - case PYBA_STATE_APPLYING: __fallthrough; - case PYBA_STATE_CACHING: __fallthrough; - case PYBA_STATE_CACHED: __fallthrough; - case PYBA_STATE_EXECUTING: __fallthrough; - case PYBA_STATE_EXECUTED: - *pdwPageId = _pageIds[PAGE_PROGRESS]; - break; - - case PYBA_STATE_APPLIED: - *pdwPageId = _pageIds[PAGE_SUCCESS]; - break; - - case PYBA_STATE_FAILED: - *pdwPageId = _pageIds[PAGE_FAILURE]; - break; - } - } - } - - BOOL WillElevate() { - static BAL_CONDITION WILL_ELEVATE_CONDITION = { - L"not WixBundleElevated and (" - /*Elevate when installing for all users*/ - L"InstallAllUsers or " - /*Elevate when installing the launcher for all users and it was not detected*/ - L"(Include_launcher and InstallLauncherAllUsers and not BlockedLauncher)" - L")", - L"" - }; - BOOL result; - - return SUCCEEDED(BalConditionEvaluate(&WILL_ELEVATE_CONDITION, _engine, &result, nullptr)) && result; - } - - BOOL IsCrtInstalled() { - if (_crtInstalledToken > 0) { - return TRUE; - } else if (_crtInstalledToken == 0) { - return FALSE; - } - - // Check whether at least CRT v10.0.10137.0 is available. - // It should only be installed as a Windows Update package, which means - // we don't need to worry about 32-bit/64-bit. - LPCWSTR crtFile = L"ucrtbase.dll"; - - DWORD cbVer = GetFileVersionInfoSizeW(crtFile, nullptr); - if (!cbVer) { - _crtInstalledToken = 0; - return FALSE; - } - - void *pData = malloc(cbVer); - if (!pData) { - _crtInstalledToken = 0; - return FALSE; - } - - if (!GetFileVersionInfoW(crtFile, 0, cbVer, pData)) { - free(pData); - _crtInstalledToken = 0; - return FALSE; - } - - VS_FIXEDFILEINFO *ffi; - UINT cb; - BOOL result = FALSE; - - if (VerQueryValueW(pData, L"\\", (LPVOID*)&ffi, &cb) && - ffi->dwFileVersionMS == 0x000A0000 && ffi->dwFileVersionLS >= 0x27990000) { - result = TRUE; - } - - free(pData); - _crtInstalledToken = result ? 1 : 0; - return result; - } - - HRESULT EvaluateConditions() { - HRESULT hr = S_OK; - BOOL result = FALSE; - - for (DWORD i = 0; i < _conditions.cConditions; ++i) { - BAL_CONDITION* pCondition = _conditions.rgConditions + i; - - hr = BalConditionEvaluate(pCondition, _engine, &result, &_failedMessage); - BalExitOnFailure(hr, "Failed to evaluate condition."); - - if (!result) { - // Hope they didn't have hidden variables in their message, because it's going in the log in plaintext. - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "%ls", _failedMessage); - - hr = E_WIXSTDBA_CONDITION_FAILED; - // todo: remove in WiX v4, in case people are relying on v3.x logging behavior - BalExitOnFailure1(hr, "Bundle condition evaluated to false: %ls", pCondition->sczCondition); - } - } - - ReleaseNullStrSecure(_failedMessage); - - LExit: - return hr; - } - - - void SetTaskbarButtonProgress(__in DWORD dwOverallPercentage) { - HRESULT hr = S_OK; - - if (_taskbarButtonOK) { - hr = _taskbarList->SetProgressValue(_hWnd, dwOverallPercentage, 100UL); - BalExitOnFailure1(hr, "Failed to set taskbar button progress to: %d%%.", dwOverallPercentage); - } - - LExit: - return; - } - - - void SetTaskbarButtonState(__in TBPFLAG tbpFlags) { - HRESULT hr = S_OK; - - if (_taskbarButtonOK) { - hr = _taskbarList->SetProgressState(_hWnd, tbpFlags); - BalExitOnFailure1(hr, "Failed to set taskbar button state.", tbpFlags); - } - - LExit: - return; - } - - - void SetProgressState(__in HRESULT hrStatus) { - TBPFLAG flag = TBPF_NORMAL; - - if (IsCanceled() || HRESULT_FROM_WIN32(ERROR_INSTALL_USEREXIT) == hrStatus) { - flag = TBPF_PAUSED; - } else if (IsRollingBack() || FAILED(hrStatus)) { - flag = TBPF_ERROR; - } - - SetTaskbarButtonState(flag); - } - - - HRESULT LoadBootstrapperBAFunctions() { - HRESULT hr = S_OK; - LPWSTR sczBafPath = nullptr; - - hr = PathRelativeToModule(&sczBafPath, L"bafunctions.dll", _hModule); - BalExitOnFailure(hr, "Failed to get path to BA function DLL."); - -#ifdef DEBUG - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "PYBA: LoadBootstrapperBAFunctions() - BA function DLL %ls", sczBafPath); -#endif - - _hBAFModule = ::LoadLibraryW(sczBafPath); - if (_hBAFModule) { - auto pfnBAFunctionCreate = reinterpret_cast(::GetProcAddress(_hBAFModule, "CreateBootstrapperBAFunction")); - BalExitOnNullWithLastError1(pfnBAFunctionCreate, hr, "Failed to get CreateBootstrapperBAFunction entry-point from: %ls", sczBafPath); - - hr = pfnBAFunctionCreate(_engine, _hBAFModule, &_baFunction); - BalExitOnFailure(hr, "Failed to create BA function."); - } -#ifdef DEBUG - else { - BalLogError(HRESULT_FROM_WIN32(::GetLastError()), "PYBA: LoadBootstrapperBAFunctions() - Failed to load DLL %ls", sczBafPath); - } -#endif - - LExit: - if (_hBAFModule && !_baFunction) { - ::FreeLibrary(_hBAFModule); - _hBAFModule = nullptr; - } - ReleaseStr(sczBafPath); - - return hr; - } - - BOOL IsCheckable(THEME_CONTROL* pControl) { - if (!pControl->sczName || !pControl->sczName[0]) { - return FALSE; - } - - if (pControl->type == THEME_CONTROL_TYPE_CHECKBOX) { - return TRUE; - } - - if (pControl->type == THEME_CONTROL_TYPE_BUTTON) { - if ((pControl->dwStyle & BS_TYPEMASK) == BS_AUTORADIOBUTTON) { - return TRUE; - } - } - - return FALSE; - } - - void SavePageSettings() { - DWORD pageId = 0; - THEME_PAGE* pPage = nullptr; - - DeterminePageId(_state, &pageId); - pPage = ThemeGetPage(_theme, pageId); - if (!pPage) { - return; - } - - for (DWORD i = 0; i < pPage->cControlIndices; ++i) { - // Loop through all the checkable controls and set a Burn variable - // with that name to true or false. - THEME_CONTROL* pControl = _theme->rgControls + pPage->rgdwControlIndices[i]; - if (IsCheckable(pControl) && ThemeControlEnabled(_theme, pControl->wId)) { - BOOL checked = ThemeIsControlChecked(_theme, pControl->wId); - _engine->SetVariableNumeric(pControl->sczName, checked ? 1 : 0); - } - - // Loop through all the editbox controls with names and set a - // Burn variable with that name to the contents. - if (THEME_CONTROL_TYPE_EDITBOX == pControl->type && pControl->sczName && *pControl->sczName) { - LPWSTR sczValue = nullptr; - ThemeGetTextControl(_theme, pControl->wId, &sczValue); - _engine->SetVariableString(pControl->sczName, sczValue); - } - } - } - - static bool IsTargetPlatformx64(__in IBootstrapperEngine* pEngine) { - WCHAR platform[8]; - DWORD platformLen = 8; - - if (FAILED(pEngine->GetVariableString(L"TargetPlatform", platform, &platformLen))) { - return S_FALSE; - } - - return ::CompareStringW(LOCALE_NEUTRAL, 0, platform, -1, L"x64", -1) == CSTR_EQUAL; - } - - static bool IsTargetPlatformARM64(__in IBootstrapperEngine* pEngine) { - WCHAR platform[8]; - DWORD platformLen = 8; - - if (FAILED(pEngine->GetVariableString(L"TargetPlatform", platform, &platformLen))) { - return S_FALSE; - } - - return ::CompareStringW(LOCALE_NEUTRAL, 0, platform, -1, L"ARM64", -1) == CSTR_EQUAL; - } - - static HRESULT LoadOptionalFeatureStatesFromKey( - __in IBootstrapperEngine* pEngine, - __in HKEY hkHive, - __in LPCWSTR subkey - ) { - HKEY hKey; - LRESULT res; - - if (IsTargetPlatformx64(pEngine) || IsTargetPlatformARM64(pEngine)) { - res = RegOpenKeyExW(hkHive, subkey, 0, KEY_READ | KEY_WOW64_64KEY, &hKey); - } else { - res = RegOpenKeyExW(hkHive, subkey, 0, KEY_READ | KEY_WOW64_32KEY, &hKey); - } - if (res == ERROR_FILE_NOT_FOUND) { - return S_FALSE; - } - if (res != ERROR_SUCCESS) { - return HRESULT_FROM_WIN32(res); - } - - for (auto p = OPTIONAL_FEATURES; p->regName; ++p) { - res = RegQueryValueExW(hKey, p->regName, nullptr, nullptr, nullptr, nullptr); - if (res == ERROR_FILE_NOT_FOUND) { - pEngine->SetVariableNumeric(p->variableName, 0); - } else if (res == ERROR_SUCCESS) { - pEngine->SetVariableNumeric(p->variableName, 1); - } else { - RegCloseKey(hKey); - return HRESULT_FROM_WIN32(res); - } - } - - RegCloseKey(hKey); - return S_OK; - } - - static HRESULT LoadTargetDirFromKey( - __in IBootstrapperEngine* pEngine, - __in HKEY hkHive, - __in LPCWSTR subkey - ) { - HKEY hKey; - LRESULT res; - DWORD dataType; - BYTE buffer[1024]; - DWORD bufferLen = sizeof(buffer); - - if (IsTargetPlatformx64(pEngine) || IsTargetPlatformARM64(pEngine)) { - res = RegOpenKeyExW(hkHive, subkey, 0, KEY_READ | KEY_WOW64_64KEY, &hKey); - } else { - res = RegOpenKeyExW(hkHive, subkey, 0, KEY_READ | KEY_WOW64_32KEY, &hKey); - } - if (res == ERROR_FILE_NOT_FOUND) { - return S_FALSE; - } - if (res != ERROR_SUCCESS) { - return HRESULT_FROM_WIN32(res); - } - - res = RegQueryValueExW(hKey, nullptr, nullptr, &dataType, buffer, &bufferLen); - if (res == ERROR_SUCCESS && dataType == REG_SZ && bufferLen < sizeof(buffer)) { - pEngine->SetVariableString(L"TargetDir", reinterpret_cast(buffer)); - } - RegCloseKey(hKey); - return HRESULT_FROM_WIN32(res); - } - - static HRESULT LoadAssociateFilesStateFromKey( - __in IBootstrapperEngine* pEngine, - __in HKEY hkHive - ) { - const LPCWSTR subkey = L"Software\\Python\\PyLauncher"; - HKEY hKey; - LRESULT res; - HRESULT hr; - - res = RegOpenKeyExW(hkHive, subkey, 0, KEY_READ | KEY_WOW64_32KEY, &hKey); - - if (res == ERROR_FILE_NOT_FOUND) { - return S_FALSE; - } - if (res != ERROR_SUCCESS) { - return HRESULT_FROM_WIN32(res); - } - - res = RegQueryValueExW(hKey, L"AssociateFiles", nullptr, nullptr, nullptr, nullptr); - if (res == ERROR_FILE_NOT_FOUND) { - hr = S_FALSE; - } else if (res == ERROR_SUCCESS) { - hr = S_OK; - } else { - hr = HRESULT_FROM_WIN32(res); - } - - RegCloseKey(hKey); - return hr; - } - - static void LoadOptionalFeatureStates(__in IBootstrapperEngine* pEngine) { - WCHAR subkeyFmt[256]; - WCHAR subkey[256]; - DWORD subkeyLen; - HRESULT hr; - HKEY hkHive; - - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Loading state of optional features"); - - // Get the registry key from the bundle, to save having to duplicate it - // in multiple places. - subkeyLen = sizeof(subkeyFmt) / sizeof(subkeyFmt[0]); - hr = pEngine->GetVariableString(L"OptionalFeaturesRegistryKey", subkeyFmt, &subkeyLen); - BalExitOnFailure(hr, "Failed to locate registry key"); - subkeyLen = sizeof(subkey) / sizeof(subkey[0]); - hr = pEngine->FormatString(subkeyFmt, subkey, &subkeyLen); - BalExitOnFailure1(hr, "Failed to format %ls", subkeyFmt); - - // Check the current user's registry for existing features - hkHive = HKEY_CURRENT_USER; - hr = LoadOptionalFeatureStatesFromKey(pEngine, hkHive, subkey); - BalExitOnFailure1(hr, "Failed to read from HKCU\\%ls", subkey); - if (hr == S_FALSE) { - // Now check the local machine registry - hkHive = HKEY_LOCAL_MACHINE; - hr = LoadOptionalFeatureStatesFromKey(pEngine, hkHive, subkey); - BalExitOnFailure1(hr, "Failed to read from HKLM\\%ls", subkey); - if (hr == S_OK) { - // Found a system-wide install, so enable these settings. - pEngine->SetVariableNumeric(L"InstallAllUsers", 1); - pEngine->SetVariableNumeric(L"CompileAll", 1); - } - } - - if (hr == S_OK) { - // Cannot change InstallAllUsersState when upgrading. While there's - // no good reason to not allow installing a per-user and an all-user - // version simultaneously, Burn can't handle the state management - // and will need to uninstall the old one. - pEngine->SetVariableString(L"InstallAllUsersState", L"disable"); - - // Get the previous install directory. This can be changed by the - // user. - subkeyLen = sizeof(subkeyFmt) / sizeof(subkeyFmt[0]); - hr = pEngine->GetVariableString(L"TargetDirRegistryKey", subkeyFmt, &subkeyLen); - BalExitOnFailure(hr, "Failed to locate registry key"); - subkeyLen = sizeof(subkey) / sizeof(subkey[0]); - hr = pEngine->FormatString(subkeyFmt, subkey, &subkeyLen); - BalExitOnFailure1(hr, "Failed to format %ls", subkeyFmt); - LoadTargetDirFromKey(pEngine, hkHive, subkey); - } - - LExit: - return; - } - - HRESULT EnsureTargetDir() { - LONGLONG installAllUsers; - LPWSTR targetDir = nullptr, defaultDir = nullptr; - HRESULT hr = BalGetStringVariable(L"TargetDir", &targetDir); - if (FAILED(hr) || !targetDir || !targetDir[0]) { - ReleaseStr(targetDir); - targetDir = nullptr; - - hr = BalGetNumericVariable(L"InstallAllUsers", &installAllUsers); - ExitOnFailure(hr, L"Failed to get install scope"); - - hr = BalGetStringVariable( - installAllUsers ? L"DefaultAllUsersTargetDir" : L"DefaultJustForMeTargetDir", - &defaultDir - ); - BalExitOnFailure(hr, "Failed to get the default install directory"); - - if (!defaultDir || !defaultDir[0]) { - BalLogError(E_INVALIDARG, "Default install directory is blank"); - } - - hr = BalFormatString(defaultDir, &targetDir); - BalExitOnFailure1(hr, "Failed to format '%ls'", defaultDir); - - hr = _engine->SetVariableString(L"TargetDir", targetDir); - BalExitOnFailure(hr, "Failed to set install target directory"); - } - LExit: - ReleaseStr(defaultDir); - ReleaseStr(targetDir); - return hr; - } - - void ValidateOperatingSystem() { - LOC_STRING *pLocString = nullptr; - - if (IsWindowsServer()) { - if (IsWindowsVersionOrGreater(10, 0, 0)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows Server 2016 or later"); - return; - } else if (IsWindowsVersionOrGreater(6, 2, 0)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2012"); - } else if (IsWindowsVersionOrGreater(6, 1, 1)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2008 R2"); - } else if (IsWindowsVersionOrGreater(6, 1, 0)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2008 R2"); - } else if (IsWindowsVersionOrGreater(6, 0, 0)) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2008"); - } else { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Server 2003 or earlier"); - } - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows Server 2016 or later is required to continue installation"); - } else { - if (IsWindows10OrGreater()) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_STANDARD, "Target OS is Windows 10 or later"); - return; - } else if (IsWindows8Point1OrGreater()) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows 8.1"); - } else if (IsWindows8OrGreater()) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows 8"); - } else if (IsWindows7OrGreater()) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows 7"); - } else if (IsWindowsVistaOrGreater()) { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows Vista"); - } else { - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Detected Windows XP or earlier"); - } - BalLog(BOOTSTRAPPER_LOG_LEVEL_ERROR, "Windows 10 or later is required to continue installation"); - } - - LocGetString(_wixLoc, L"#(loc.FailureOldOS)", &pLocString); - if (pLocString && pLocString->wzText) { - BalFormatString(pLocString->wzText, &_failedMessage); - } - - _hrFinal = E_WIXSTDBA_CONDITION_FAILED; - } - -public: - // - // Constructor - initialize member variables. - // - PythonBootstrapperApplication( - __in HMODULE hModule, - __in BOOL fPrereq, - __in HRESULT hrHostInitialization, - __in IBootstrapperEngine* pEngine, - __in const BOOTSTRAPPER_COMMAND* pCommand - ) : CBalBaseBootstrapperApplication(pEngine, pCommand, 3, 3000) { - _hModule = hModule; - memcpy_s(&_command, sizeof(_command), pCommand, sizeof(BOOTSTRAPPER_COMMAND)); - - LONGLONG llInstalled = 0; - HRESULT hr = BalGetNumericVariable(L"WixBundleInstalled", &llInstalled); - if (SUCCEEDED(hr) && BOOTSTRAPPER_RESUME_TYPE_REBOOT != _command.resumeType && 0 < llInstalled && BOOTSTRAPPER_ACTION_INSTALL == _command.action) { - _command.action = BOOTSTRAPPER_ACTION_MODIFY; - } else if (0 == llInstalled && (BOOTSTRAPPER_ACTION_MODIFY == _command.action || BOOTSTRAPPER_ACTION_REPAIR == _command.action)) { - _command.action = BOOTSTRAPPER_ACTION_INSTALL; - } - - _plannedAction = BOOTSTRAPPER_ACTION_UNKNOWN; - - - // When resuming from restart doing some install-like operation, try to find the package that forced the - // restart. We'll use this information during planning. - _nextPackageAfterRestart = nullptr; - - if (BOOTSTRAPPER_RESUME_TYPE_REBOOT == _command.resumeType && BOOTSTRAPPER_ACTION_UNINSTALL < _command.action) { - // Ensure the forced restart package variable is null when it is an empty string. - HRESULT hr = BalGetStringVariable(L"WixBundleForcedRestartPackage", &_nextPackageAfterRestart); - if (FAILED(hr) || !_nextPackageAfterRestart || !*_nextPackageAfterRestart) { - ReleaseNullStr(_nextPackageAfterRestart); - } - } - - _crtInstalledToken = -1; - pEngine->SetVariableNumeric(L"CRTInstalled", IsCrtInstalled() ? 1 : 0); - - _wixLoc = nullptr; - memset(&_bundle, 0, sizeof(_bundle)); - memset(&_conditions, 0, sizeof(_conditions)); - _confirmCloseMessage = nullptr; - _failedMessage = nullptr; - - _language = nullptr; - _theme = nullptr; - memset(_pageIds, 0, sizeof(_pageIds)); - _hUiThread = nullptr; - _registered = FALSE; - _hWnd = nullptr; - - _state = PYBA_STATE_INITIALIZING; - _visiblePageId = 0; - _installPage = PAGE_LOADING; - _hrFinal = hrHostInitialization; - - _downgradingOtherVersion = FALSE; - _restartResult = BOOTSTRAPPER_APPLY_RESTART_NONE; - _restartRequired = FALSE; - _allowRestart = FALSE; - - _suppressDowngradeFailure = FALSE; - _suppressRepair = FALSE; - _modifying = FALSE; - _upgrading = FALSE; - - _overridableVariables = nullptr; - _taskbarList = nullptr; - _taskbarButtonCreatedMessage = UINT_MAX; - _taskbarButtonOK = FALSE; - _showingInternalUIThisPackage = FALSE; - - _suppressPaint = FALSE; - - pEngine->AddRef(); - _engine = pEngine; - - _hBAFModule = nullptr; - _baFunction = nullptr; - } - - - // - // Destructor - release member variables. - // - ~PythonBootstrapperApplication() { - AssertSz(!::IsWindow(_hWnd), "Window should have been destroyed before destructor."); - AssertSz(!_theme, "Theme should have been released before destructor."); - - ReleaseObject(_taskbarList); - ReleaseDict(_overridableVariables); - ReleaseStr(_failedMessage); - ReleaseStr(_confirmCloseMessage); - BalConditionsUninitialize(&_conditions); - BalInfoUninitialize(&_bundle); - LocFree(_wixLoc); - - ReleaseStr(_language); - ReleaseStr(_nextPackageAfterRestart); - ReleaseNullObject(_engine); - - if (_hBAFModule) { - ::FreeLibrary(_hBAFModule); - _hBAFModule = nullptr; - } - } - -private: - HMODULE _hModule; - BOOTSTRAPPER_COMMAND _command; - IBootstrapperEngine* _engine; - BOOTSTRAPPER_ACTION _plannedAction; - - LPWSTR _nextPackageAfterRestart; - - WIX_LOCALIZATION* _wixLoc; - BAL_INFO_BUNDLE _bundle; - BAL_CONDITIONS _conditions; - LPWSTR _failedMessage; - LPWSTR _confirmCloseMessage; - - LPWSTR _language; - THEME* _theme; - DWORD _pageIds[countof(PAGE_NAMES)]; - HANDLE _hUiThread; - BOOL _registered; - HWND _hWnd; - - PYBA_STATE _state; - HRESULT _hrFinal; - DWORD _visiblePageId; - PAGE _installPage; - - BOOL _startedExecution; - DWORD _calculatedCacheProgress; - DWORD _calculatedExecuteProgress; - - BOOL _downgradingOtherVersion; - BOOTSTRAPPER_APPLY_RESTART _restartResult; - BOOL _restartRequired; - BOOL _allowRestart; - - BOOL _suppressDowngradeFailure; - BOOL _suppressRepair; - BOOL _modifying; - BOOL _upgrading; - - int _crtInstalledToken; - - STRINGDICT_HANDLE _overridableVariables; - - ITaskbarList3* _taskbarList; - UINT _taskbarButtonCreatedMessage; - BOOL _taskbarButtonOK; - BOOL _showingInternalUIThisPackage; - - BOOL _suppressPaint; - - HMODULE _hBAFModule; - IBootstrapperBAFunction* _baFunction; -}; - -// -// CreateBootstrapperApplication - creates a new IBootstrapperApplication object. -// -HRESULT CreateBootstrapperApplication( - __in HMODULE hModule, - __in BOOL fPrereq, - __in HRESULT hrHostInitialization, - __in IBootstrapperEngine* pEngine, - __in const BOOTSTRAPPER_COMMAND* pCommand, - __out IBootstrapperApplication** ppApplication - ) { - HRESULT hr = S_OK; - - if (fPrereq) { - hr = E_INVALIDARG; - ExitWithLastError(hr, "Failed to create UI thread."); - } - - PythonBootstrapperApplication* pApplication = nullptr; - - pApplication = new PythonBootstrapperApplication(hModule, fPrereq, hrHostInitialization, pEngine, pCommand); - ExitOnNull(pApplication, hr, E_OUTOFMEMORY, "Failed to create new standard bootstrapper application object."); - - *ppApplication = pApplication; - pApplication = nullptr; - -LExit: - ReleaseObject(pApplication); - return hr; -} diff --git a/Tools/msi/bundle/bootstrap/pch.cpp b/Tools/msi/bundle/bootstrap/pch.cpp deleted file mode 100644 index 1d9f38c57d63f19..000000000000000 --- a/Tools/msi/bundle/bootstrap/pch.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "pch.h" diff --git a/Tools/msi/bundle/bootstrap/pch.h b/Tools/msi/bundle/bootstrap/pch.h deleted file mode 100644 index 6d0974b34c61e79..000000000000000 --- a/Tools/msi/bundle/bootstrap/pch.h +++ /dev/null @@ -1,60 +0,0 @@ -//------------------------------------------------------------------------------------------------- -// -// Copyright (c) 2004, Outercurve Foundation. -// This software is released under Microsoft Reciprocal License (MS-RL). -// The license and further copyright text can be found in the file -// LICENSE.TXT at the root directory of the distribution. -// -// -// -// Precompiled header for standard bootstrapper application. -// -//------------------------------------------------------------------------------------------------- - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "dutil.h" -#include "memutil.h" -#include "dictutil.h" -#include "dirutil.h" -#include "fileutil.h" -#include "locutil.h" -#include "logutil.h" -#include "pathutil.h" -#include "resrutil.h" -#include "shelutil.h" -#include "strutil.h" -#include "thmutil.h" -#include "uriutil.h" -#include "xmlutil.h" - -#include "IBootstrapperEngine.h" -#include "IBootstrapperApplication.h" - -#include "BalBaseBootstrapperApplication.h" -#include "balinfo.h" -#include "balcondition.h" - -HRESULT CreateBootstrapperApplication( - __in HMODULE hModule, - __in BOOL fPrereq, - __in HRESULT hrHostInitialization, - __in IBootstrapperEngine* pEngine, - __in const BOOTSTRAPPER_COMMAND* pCommand, - __out IBootstrapperApplication** ppApplication -); - -#include "IBootstrapperBAFunction.h" - diff --git a/Tools/msi/bundle/bootstrap/pythonba.cpp b/Tools/msi/bundle/bootstrap/pythonba.cpp deleted file mode 100644 index 0ce45ad31d958a5..000000000000000 --- a/Tools/msi/bundle/bootstrap/pythonba.cpp +++ /dev/null @@ -1,76 +0,0 @@ -//------------------------------------------------------------------------------------------------- -// -// Copyright (c) 2004, Outercurve Foundation. -// This software is released under Microsoft Reciprocal License (MS-RL). -// The license and further copyright text can be found in the file -// LICENSE.TXT at the root directory of the distribution. -// -// -// -// Setup chainer/bootstrapper standard UI for WiX toolset. -// -//------------------------------------------------------------------------------------------------- - -#include "pch.h" - -static HINSTANCE vhInstance = NULL; - -extern "C" BOOL WINAPI DllMain( - IN HINSTANCE hInstance, - IN DWORD dwReason, - IN LPVOID /* pvReserved */ - ) -{ - switch(dwReason) - { - case DLL_PROCESS_ATTACH: - ::DisableThreadLibraryCalls(hInstance); - vhInstance = hInstance; - break; - - case DLL_PROCESS_DETACH: - vhInstance = NULL; - break; - } - - return TRUE; -} - - -extern "C" HRESULT WINAPI BootstrapperApplicationCreate( - __in IBootstrapperEngine* pEngine, - __in const BOOTSTRAPPER_COMMAND* pCommand, - __out IBootstrapperApplication** ppApplication - ) -{ - HRESULT hr = S_OK; - - BalInitialize(pEngine); - - hr = CreateBootstrapperApplication(vhInstance, FALSE, S_OK, pEngine, pCommand, ppApplication); - BalExitOnFailure(hr, "Failed to create bootstrapper application interface."); - -LExit: - return hr; -} - - -extern "C" void WINAPI BootstrapperApplicationDestroy() -{ - BalUninitialize(); -} - - -extern "C" HRESULT WINAPI MbaPrereqBootstrapperApplicationCreate( - __in HRESULT hrHostInitialization, - __in IBootstrapperEngine* pEngine, - __in const BOOTSTRAPPER_COMMAND* pCommand, - __out IBootstrapperApplication** ppApplication - ) -{ - return E_NOTIMPL; -} - - -extern "C" void WINAPI MbaPrereqBootstrapperApplicationDestroy() -{ } diff --git a/Tools/msi/bundle/bootstrap/pythonba.def b/Tools/msi/bundle/bootstrap/pythonba.def deleted file mode 100644 index 29b3fa50dfa9f70..000000000000000 --- a/Tools/msi/bundle/bootstrap/pythonba.def +++ /dev/null @@ -1,18 +0,0 @@ -;------------------------------------------------------------------------------------------------- -; -; Copyright (c) 2004, Outercurve Foundation. -; This software is released under Microsoft Reciprocal License (MS-RL). -; The license and further copyright text can be found in the file -; LICENSE.TXT at the root directory of the distribution. -; -; -; -; WiX Standard Bootstrapper Application DLL entry points. -; -;------------------------------------------------------------------------------------------------- - -EXPORTS - BootstrapperApplicationCreate - BootstrapperApplicationDestroy - MbaPrereqBootstrapperApplicationCreate - MbaPrereqBootstrapperApplicationDestroy diff --git a/Tools/msi/bundle/bootstrap/pythonba.sln b/Tools/msi/bundle/bootstrap/pythonba.sln deleted file mode 100644 index bf43fed9018e4f3..000000000000000 --- a/Tools/msi/bundle/bootstrap/pythonba.sln +++ /dev/null @@ -1,22 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2013 -VisualStudioVersion = 12.0.30501.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythonba", "pythonba.vcxproj", "{7A09B132-B3EE-499B-A700-A4B2157FEA3D}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Release|Win32 = Release|Win32 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Debug|Win32.ActiveCfg = Debug|Win32 - {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Debug|Win32.Build.0 = Debug|Win32 - {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Release|Win32.ActiveCfg = Release|Win32 - {7A09B132-B3EE-499B-A700-A4B2157FEA3D}.Release|Win32.Build.0 = Release|Win32 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/Tools/msi/bundle/bootstrap/pythonba.vcxproj b/Tools/msi/bundle/bootstrap/pythonba.vcxproj deleted file mode 100644 index 4b38582f0d31e51..000000000000000 --- a/Tools/msi/bundle/bootstrap/pythonba.vcxproj +++ /dev/null @@ -1,73 +0,0 @@ - - - - - - Debug - Win32 - - - Release - Win32 - - - - Release - Win32 - v145 - v143 - v142 - v141 - {7A09B132-B3EE-499B-A700-A4B2157FEA3D} - PythonBA - - - - - DynamicLibrary - Unicode - $(PySourcePath)PCbuild\obj\ - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\msi_$(ProjectName)\ - $(IntDir.Replace(`\\`, `\`)) - $(IntDir) - - - - - ARM64=1;%(PreprocessorDefinitions) - _CRT_STDIO_LEGACY_WIDE_SPECIFIERS=1;%(PreprocessorDefinitions) - $(WixInstallPath)sdk\inc - Use - pch.h - MultiThreaded - - - comctl32.lib;gdiplus.lib;msimg32.lib;shlwapi.lib;wininet.lib;dutil.lib;balutil.lib;version.lib;uxtheme.lib;%(AdditionalDependencies) - $(WixInstallPath)sdk\vs2017\lib\x86 - pythonba.def - true - - - - - - - Create - - - - - - - - - - - diff --git a/Tools/msi/bundle/bootstrap/resource.h b/Tools/msi/bundle/bootstrap/resource.h deleted file mode 100644 index d951e651f6d20d7..000000000000000 --- a/Tools/msi/bundle/bootstrap/resource.h +++ /dev/null @@ -1,25 +0,0 @@ -//------------------------------------------------------------------------------------------------- -// -// Copyright (c) 2004, Outercurve Foundation. -// This software is released under Microsoft Reciprocal License (MS-RL). -// The license and further copyright text can be found in the file -// LICENSE.TXT at the root directory of the distribution. -// -//------------------------------------------------------------------------------------------------- - -//{{NO_DEPENDENCIES}} -// Microsoft Visual C++ generated include file. -// -#define IDC_STATIC -1 - - -// Next default values for new objects -// -#ifdef APSTUDIO_INVOKED -#ifndef APSTUDIO_READONLY_SYMBOLS -#define _APS_NEXT_RESOURCE_VALUE 102 -#define _APS_NEXT_COMMAND_VALUE 40001 -#define _APS_NEXT_CONTROL_VALUE 1003 -#define _APS_NEXT_SYMED_VALUE 101 -#endif -#endif diff --git a/Tools/msi/bundle/bundle.targets b/Tools/msi/bundle/bundle.targets deleted file mode 100644 index cb3effb4434843a..000000000000000 --- a/Tools/msi/bundle/bundle.targets +++ /dev/null @@ -1,129 +0,0 @@ - - - - 2.0 - Bundle - - Release - 1132;1135;1140 - $(OutputName)-$(PythonVersion) - $(OutputName)-$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber).$(RevisionNumber) - $(OutputName)-amd64 - $(OutputName)-arm64 - $(OutputName)-$(OutputSuffix) - $(OutputName)-d - $(OutputName) - - $(OutputPath)en-us\ - $(OutputPath) - - - $(DownloadUrlBase.TrimEnd(`/`))/{version}/{arch}{releasename}/{msi} - $(DefineConstants);DownloadUrl=$(DownloadUrl.Replace(`{version}`, `$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)`).Replace(`{arch}`, `$(ArchName)`).Replace(`{releasename}`, `$(ReleaseLevelName)`).Replace(`{msi}`, `{2}`)) - $(DefineConstants);DownloadUrl={2} - - - - - WixUtilExtension - WixUtilExtension - - - WixDependencyExtension - WixDependencyExtension - - - WixBalExtension - WixBalExtension - - - - - - - - - - - - - - - - - - - - - - - BuildForRelease=$(BuildForRelease) - - - - - - - - - - - - - - - - - - - Build - - - - - Rebuild - - - - - - - - - - - - - - - - $(DefineConstants);BootstrapApp=$(BootstrapAppPath) - - - - - - - - - $(DefineConstants);DefaultWxl=Default.wxl - - - - - - - - - $(DefineConstants);DefaultWxl=$(IntermediateOutputPath)Default.wxl - - - - - diff --git a/Tools/msi/bundle/bundle.wxl b/Tools/msi/bundle/bundle.wxl deleted file mode 100644 index d7a65c48dc4dc53..000000000000000 --- a/Tools/msi/bundle/bundle.wxl +++ /dev/null @@ -1,7 +0,0 @@ - - - C Runtime Update (KB2999226) - Precompiling standard library - Precompiling standard library (-O) - Precompiling standard library (-OO) - diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs deleted file mode 100644 index 3fcb00553f5edd3..000000000000000 --- a/Tools/msi/bundle/bundle.wxs +++ /dev/null @@ -1,128 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/bundle/full.wixproj b/Tools/msi/bundle/full.wixproj deleted file mode 100644 index bdbdd8e175c0332..000000000000000 --- a/Tools/msi/bundle/full.wixproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - {3E204ADD-238D-4D10-852C-4F859325C839} - python - full - - - - - - - $(DefineConstants); - CompressMSI=yes; - CompressPDB=yes; - CompressMSI_D=yes; - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/core.wxs b/Tools/msi/bundle/packagegroups/core.wxs deleted file mode 100644 index eb3d0b7733cf0fd..000000000000000 --- a/Tools/msi/bundle/packagegroups/core.wxs +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/crt.wxs b/Tools/msi/bundle/packagegroups/crt.wxs deleted file mode 100644 index f931da1403b2278..000000000000000 --- a/Tools/msi/bundle/packagegroups/crt.wxs +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/dev.wxs b/Tools/msi/bundle/packagegroups/dev.wxs deleted file mode 100644 index 4284dba2c205b73..000000000000000 --- a/Tools/msi/bundle/packagegroups/dev.wxs +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/doc.wxs b/Tools/msi/bundle/packagegroups/doc.wxs deleted file mode 100644 index 6639ff5c25b24be..000000000000000 --- a/Tools/msi/bundle/packagegroups/doc.wxs +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/exe.wxs b/Tools/msi/bundle/packagegroups/exe.wxs deleted file mode 100644 index 79464c4aca66222..000000000000000 --- a/Tools/msi/bundle/packagegroups/exe.wxs +++ /dev/null @@ -1,64 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/freethreaded.wxs b/Tools/msi/bundle/packagegroups/freethreaded.wxs deleted file mode 100644 index 121ca34ab661578..000000000000000 --- a/Tools/msi/bundle/packagegroups/freethreaded.wxs +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/launcher.wxs b/Tools/msi/bundle/packagegroups/launcher.wxs deleted file mode 100644 index 080598a0a486ef4..000000000000000 --- a/Tools/msi/bundle/packagegroups/launcher.wxs +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/lib.wxs b/Tools/msi/bundle/packagegroups/lib.wxs deleted file mode 100644 index 0b3fbc00c4a35db..000000000000000 --- a/Tools/msi/bundle/packagegroups/lib.wxs +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/packageinstall.wxs b/Tools/msi/bundle/packagegroups/packageinstall.wxs deleted file mode 100644 index e5e7d4d1a94cd1f..000000000000000 --- a/Tools/msi/bundle/packagegroups/packageinstall.wxs +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/pip.wxs b/Tools/msi/bundle/packagegroups/pip.wxs deleted file mode 100644 index 201a6c445b23691..000000000000000 --- a/Tools/msi/bundle/packagegroups/pip.wxs +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/postinstall.wxs b/Tools/msi/bundle/packagegroups/postinstall.wxs deleted file mode 100644 index 24c72e6357ab082..000000000000000 --- a/Tools/msi/bundle/packagegroups/postinstall.wxs +++ /dev/null @@ -1,109 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/bundle/packagegroups/tcltk.wxs b/Tools/msi/bundle/packagegroups/tcltk.wxs deleted file mode 100644 index 0d029a90cd966ed..000000000000000 --- a/Tools/msi/bundle/packagegroups/tcltk.wxs +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/packagegroups/test.wxs b/Tools/msi/bundle/packagegroups/test.wxs deleted file mode 100644 index 32acaef64efea07..000000000000000 --- a/Tools/msi/bundle/packagegroups/test.wxs +++ /dev/null @@ -1,62 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/releaselocal.wixproj b/Tools/msi/bundle/releaselocal.wixproj deleted file mode 100644 index 0c3dee7ad81ea19..000000000000000 --- a/Tools/msi/bundle/releaselocal.wixproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - {FCD43AC9-969F-49A1-8AC5-EDC27599D1EB} - python - - - - - - - - $(DefineConstants); - CompressMSI=yes; - CompressPDB=no; - CompressMSI_D=no - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/releaseweb.wixproj b/Tools/msi/bundle/releaseweb.wixproj deleted file mode 100644 index 350c735878c0b4a..000000000000000 --- a/Tools/msi/bundle/releaseweb.wixproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - {71CDE213-CB39-4BD9-B89D-BBB878689144} - python - webinstall - - - - - - - $(DefineConstants); - CompressMSI=no; - CompressPDB=no; - CompressMSI_D=no - - - - - \ No newline at end of file diff --git a/Tools/msi/bundle/snapshot.wixproj b/Tools/msi/bundle/snapshot.wixproj deleted file mode 100644 index cc450432990635e..000000000000000 --- a/Tools/msi/bundle/snapshot.wixproj +++ /dev/null @@ -1,26 +0,0 @@ - - - - {8A4A1162-4BF9-4FF6-9A98-315F01E44932} - python - - - - - - - - $(DefineConstants);CompressMSI=no; - - - $(DefineConstants);CompressMSI=yes; - - - $(DefineConstants); - CompressPDB=no; - CompressMSI_D=no; - - - - - \ No newline at end of file diff --git a/Tools/msi/common.wxs b/Tools/msi/common.wxs deleted file mode 100644 index 73da474e4181f11..000000000000000 --- a/Tools/msi/common.wxs +++ /dev/null @@ -1,138 +0,0 @@ - - - - - - - - - - - - - OPTIONALFEATURESREGISTRYKEY - - - - - - - - - - - - - - - - - - - - - Installed OR NOT MISSING_CORE - - - - Installed OR NOT DOWNGRADE - Installed OR TARGETDIR OR Suppress_TARGETDIR_Check - - - UPGRADE - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/common_en-US.wxl_template b/Tools/msi/common_en-US.wxl_template deleted file mode 100644 index adb8f40be9a8e23..000000000000000 --- a/Tools/msi/common_en-US.wxl_template +++ /dev/null @@ -1,18 +0,0 @@ - - - 1033 - en-us - Python {{ShortVersion}} - Python {{LongVersion}} ({{Bitness}}) - Python {{LongVersion}} !(loc.Descriptor) ({{Bitness}}) - Python {{LongVersion}} !(loc.Descriptor) ({{Bitness}}) - Python {{LongVersion}} !(loc.Descriptor) ({{Bitness}} symbols) - Python {{LongVersion}} !(loc.Descriptor) ({{Bitness}} symbols) - Python {{LongVersion}} !(loc.Descriptor) ({{Bitness}} debug) - Python {{LongVersion}} !(loc.Descriptor) ({{Bitness}} debug) - Python Software Foundation - A newer version of !(loc.ProductName) is already installed. - An incorrect version of a prerequisite package is installed. Please uninstall any other versions of !(loc.ProductName) and try installing this again. - The TARGETDIR variable must be provided when invoking this installer. - https://www.python.org/ - diff --git a/Tools/msi/core/core.wixproj b/Tools/msi/core/core.wixproj deleted file mode 100644 index 68e8bab3109dcce..000000000000000 --- a/Tools/msi/core/core.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {1B4502D5-B627-4F50-ABEA-4CC5A8E88265} - 2.0 - core - Package - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/core/core.wxs b/Tools/msi/core/core.wxs deleted file mode 100644 index 21a8bcdf9710174..000000000000000 --- a/Tools/msi/core/core.wxs +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/Tools/msi/core/core_d.wixproj b/Tools/msi/core/core_d.wixproj deleted file mode 100644 index 5b296bf35f95807..000000000000000 --- a/Tools/msi/core/core_d.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {D3677DCF-098A-4398-9FA5-8E74AC37E0DF} - 2.0 - core_d - Package - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/core/core_d.wxs b/Tools/msi/core/core_d.wxs deleted file mode 100644 index 1cee0973aac312f..000000000000000 --- a/Tools/msi/core/core_d.wxs +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - diff --git a/Tools/msi/core/core_en-US.wxl b/Tools/msi/core/core_en-US.wxl deleted file mode 100644 index 7977470d7c1c0eb..000000000000000 --- a/Tools/msi/core/core_en-US.wxl +++ /dev/null @@ -1,5 +0,0 @@ - - - Core Interpreter - core - diff --git a/Tools/msi/core/core_files.wxs b/Tools/msi/core/core_files.wxs deleted file mode 100644 index 8b21501078ea2e6..000000000000000 --- a/Tools/msi/core/core_files.wxs +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/core/core_pdb.wixproj b/Tools/msi/core/core_pdb.wixproj deleted file mode 100644 index 9c8838970b6fee9..000000000000000 --- a/Tools/msi/core/core_pdb.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {E98E7539-64E7-4DCE-AACD-01E3ADE40EFD} - 2.0 - core_pdb - Package - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/core/core_pdb.wxs b/Tools/msi/core/core_pdb.wxs deleted file mode 100644 index 1fe7d1e7a944cb9..000000000000000 --- a/Tools/msi/core/core_pdb.wxs +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - diff --git a/Tools/msi/csv_to_wxs.py b/Tools/msi/csv_to_wxs.py deleted file mode 100644 index 235c8f8b0be5812..000000000000000 --- a/Tools/msi/csv_to_wxs.py +++ /dev/null @@ -1,127 +0,0 @@ -''' -Processes a CSV file containing a list of files into a WXS file with -components for each listed file. - -The CSV columns are: - source of file, target for file, group name - -Usage:: - py txt_to_wxs.py [path to file list .csv] [path to destination .wxs] - -This is necessary to handle structures where some directories only -contain other directories. MSBuild is not able to generate the -Directory entries in the WXS file correctly, as it operates on files. -Python, however, can easily fill in the gap. -''' - -__author__ = "Steve Dower " - -import csv -import re -import sys - -from collections import defaultdict -from itertools import chain, zip_longest -from pathlib import PureWindowsPath -from uuid import uuid1 - -ID_CHAR_SUBS = { - '-': '_', - '+': '_P', -} - -def make_id(path): - return re.sub( - r'[^A-Za-z0-9_.]', - lambda m: ID_CHAR_SUBS.get(m.group(0), '_'), - str(path).rstrip('/\\'), - flags=re.I - ) - -DIRECTORIES = set() - -def main(file_source, install_target): - with open(file_source, 'r', newline='') as f: - files = list(csv.reader(f)) - - assert len(files) == len(set(make_id(f[1]) for f in files)), "Duplicate file IDs exist" - - directories = defaultdict(set) - cache_directories = defaultdict(set) - groups = defaultdict(list) - for source, target, group, disk_id, condition in files: - target = PureWindowsPath(target) - groups[group].append((source, target, disk_id, condition)) - - if target.suffix.lower() in {".py", ".pyw"}: - cache_directories[group].add(target.parent) - - for dirname in target.parents: - parent = make_id(dirname.parent) - if parent and parent != '.': - directories[parent].add(dirname.name) - - lines = [ - '', - ' ', - ] - for dir_parent in sorted(directories): - lines.append(' '.format(dir_parent)) - for dir_name in sorted(directories[dir_parent]): - lines.append(' '.format(dir_parent, make_id(dir_name), dir_name)) - lines.append(' ') - for dir_parent in (make_id(d) for group in cache_directories.values() for d in group): - lines.append(' '.format(dir_parent)) - lines.append(' '.format(dir_parent)) - lines.append(' ') - lines.append(' ') - - for group in sorted(groups): - lines.extend([ - ' ', - ' '.format(group), - ]) - for source, target, disk_id, condition in groups[group]: - lines.append(' '.format(make_id(target), make_id(target.parent))) - if condition: - lines.append(' {}'.format(condition)) - - if disk_id: - lines.append(' '.format(make_id(target), target.name, source, disk_id)) - else: - lines.append(' '.format(make_id(target), target.name, source)) - lines.append(' ') - - create_folders = {make_id(p) + "___pycache__" for p in cache_directories[group]} - remove_folders = {make_id(p2) for p1 in cache_directories[group] for p2 in chain((p1,), p1.parents)} - create_folders.discard(".") - remove_folders.discard(".") - if create_folders or remove_folders: - lines.append(' '.format(group, uuid1())) - lines.extend(' '.format(p) for p in create_folders) - lines.extend(' '.format(p) for p in create_folders) - lines.extend(' '.format(p) for p in create_folders | remove_folders) - lines.append(' ') - - lines.extend([ - ' ', - ' ', - ]) - lines.append('') - - # Check if the file matches. If so, we don't want to touch it so - # that we can skip rebuilding. - try: - with open(install_target, 'r') as f: - if all(x.rstrip('\r\n') == y for x, y in zip_longest(f, lines)): - print('File is up to date') - return - except IOError: - pass - - with open(install_target, 'w') as f: - f.writelines(line + '\n' for line in lines) - print('Wrote {} lines to {}'.format(len(lines), install_target)) - -if __name__ == '__main__': - main(sys.argv[1], sys.argv[2]) diff --git a/Tools/msi/dev/dev.wixproj b/Tools/msi/dev/dev.wixproj deleted file mode 100644 index 4052e4b5368273a..000000000000000 --- a/Tools/msi/dev/dev.wixproj +++ /dev/null @@ -1,29 +0,0 @@ - - - - {5F23F608-D74B-4259-A0CE-8DC65CC7FE53} - 2.0 - dev - Package - - - - - - - - - - - - $(PySourcePath) - !(bindpath.src) - $(PySourcePath) - - dev_include - - - - - diff --git a/Tools/msi/dev/dev.wxs b/Tools/msi/dev/dev.wxs deleted file mode 100644 index 15a08364bead043..000000000000000 --- a/Tools/msi/dev/dev.wxs +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/dev/dev_d.wixproj b/Tools/msi/dev/dev_d.wixproj deleted file mode 100644 index b3b05326d41963f..000000000000000 --- a/Tools/msi/dev/dev_d.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {C11B4945-76BD-4137-B2E3-649460117A77} - 2.0 - dev_d - Package - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/dev/dev_d.wxs b/Tools/msi/dev/dev_d.wxs deleted file mode 100644 index 20bf9ee256f2f85..000000000000000 --- a/Tools/msi/dev/dev_d.wxs +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/Tools/msi/dev/dev_en-US.wxl b/Tools/msi/dev/dev_en-US.wxl deleted file mode 100644 index 2546e13e47a091d..000000000000000 --- a/Tools/msi/dev/dev_en-US.wxl +++ /dev/null @@ -1,5 +0,0 @@ - - - Development Libraries - dev - diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs deleted file mode 100644 index a9039d03f5f6fa1..000000000000000 --- a/Tools/msi/dev/dev_files.wxs +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/doc/doc.wixproj b/Tools/msi/doc/doc.wixproj deleted file mode 100644 index 83687968faebea8..000000000000000 --- a/Tools/msi/doc/doc.wixproj +++ /dev/null @@ -1,37 +0,0 @@ - - - - {0D62A2BB-5F71-4447-8C8C-9708407B3674} - 2.0 - doc - Package - - ICE43 - - - - $(PySourcePath)Doc\build\html\ - $(DocHtmlPath)\ - - - - doc_html - - - - - - - - - $(DocHtmlPath) - !(bindpath.doc_html) - $(DocHtmlPath)..\ - Doc_ - doc_html - - - - - \ No newline at end of file diff --git a/Tools/msi/doc/doc.wxs b/Tools/msi/doc/doc.wxs deleted file mode 100644 index 49798c752633fb2..000000000000000 --- a/Tools/msi/doc/doc.wxs +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/doc/doc_en-US.wxl_template b/Tools/msi/doc/doc_en-US.wxl_template deleted file mode 100644 index 809556e29fb9b13..000000000000000 --- a/Tools/msi/doc/doc_en-US.wxl_template +++ /dev/null @@ -1,7 +0,0 @@ - - - doc - Documentation - Python {{ShortVersion}} Manuals ({{Bitness}}) - View the !(loc.ProductName) documentation. - diff --git a/Tools/msi/exe/exe.wixproj b/Tools/msi/exe/exe.wixproj deleted file mode 100644 index 592c8d2f65ae6e0..000000000000000 --- a/Tools/msi/exe/exe.wixproj +++ /dev/null @@ -1,41 +0,0 @@ - - - - {6BD53305-B03E-49DC-85FB-5551B8CCC843} - 2.0 - exe - Package - - - - ICE43 - - - - $(DefineConstants);Include_Vcruntime140_1_dll=1 - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/exe/exe.wxs b/Tools/msi/exe/exe.wxs deleted file mode 100644 index 05a8d775193e38f..000000000000000 --- a/Tools/msi/exe/exe.wxs +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/exe/exe_d.wixproj b/Tools/msi/exe/exe_d.wixproj deleted file mode 100644 index cf085bed4ddcf95..000000000000000 --- a/Tools/msi/exe/exe_d.wixproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - {B1CA739C-8DB0-403B-9010-D79507507CE9} - 2.0 - exe_d - Package - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/exe/exe_d.wxs b/Tools/msi/exe/exe_d.wxs deleted file mode 100644 index 3b327bc26d7893c..000000000000000 --- a/Tools/msi/exe/exe_d.wxs +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/Tools/msi/exe/exe_en-US.wxl_template b/Tools/msi/exe/exe_en-US.wxl_template deleted file mode 100644 index 4cd9e3e654b2a1e..000000000000000 --- a/Tools/msi/exe/exe_en-US.wxl_template +++ /dev/null @@ -1,8 +0,0 @@ - - - Executables - executable - Python {{ShortVersion}} ({{Bitness}}) - Launches the !(loc.ProductName) interpreter. - https://www.python.org/ - diff --git a/Tools/msi/exe/exe_files.wxs b/Tools/msi/exe/exe_files.wxs deleted file mode 100644 index 15bf6a06634bf20..000000000000000 --- a/Tools/msi/exe/exe_files.wxs +++ /dev/null @@ -1,84 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/exe/exe_pdb.wixproj b/Tools/msi/exe/exe_pdb.wixproj deleted file mode 100644 index bf1213e9d499680..000000000000000 --- a/Tools/msi/exe/exe_pdb.wixproj +++ /dev/null @@ -1,21 +0,0 @@ - - - - {4A1F7045-8EE2-4276-ABB8-5E0C40E5F38B} - 2.0 - exe_pdb - Package - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/exe/exe_pdb.wxs b/Tools/msi/exe/exe_pdb.wxs deleted file mode 100644 index f706cf3b0e0cf3f..000000000000000 --- a/Tools/msi/exe/exe_pdb.wxs +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/Tools/msi/exe/exe_reg.wxs b/Tools/msi/exe/exe_reg.wxs deleted file mode 100644 index 4443c2155485bec..000000000000000 --- a/Tools/msi/exe/exe_reg.wxs +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/freethreaded/freethreaded.wixproj b/Tools/msi/freethreaded/freethreaded.wixproj deleted file mode 100644 index 0b4bd055d779773..000000000000000 --- a/Tools/msi/freethreaded/freethreaded.wixproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - {1B4502D5-B627-4F50-ABEA-4CC5A8E88265} - 2.0 - freethreaded - Package - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded.wxs b/Tools/msi/freethreaded/freethreaded.wxs deleted file mode 100644 index 063aa28bf09fceb..000000000000000 --- a/Tools/msi/freethreaded/freethreaded.wxs +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/freethreaded/freethreaded_d.wixproj b/Tools/msi/freethreaded/freethreaded_d.wixproj deleted file mode 100644 index e1563d4f907126f..000000000000000 --- a/Tools/msi/freethreaded/freethreaded_d.wixproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - {D3677DCF-098A-4398-9FA5-8E74AC37E0DF} - 2.0 - freethreaded_d - Package - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded_d.wxs b/Tools/msi/freethreaded/freethreaded_d.wxs deleted file mode 100644 index cddf22a6c803d3e..000000000000000 --- a/Tools/msi/freethreaded/freethreaded_d.wxs +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/Tools/msi/freethreaded/freethreaded_en-US.wxl_template b/Tools/msi/freethreaded/freethreaded_en-US.wxl_template deleted file mode 100644 index b9747eb256d24b1..000000000000000 --- a/Tools/msi/freethreaded/freethreaded_en-US.wxl_template +++ /dev/null @@ -1,8 +0,0 @@ - - - Freethreaded Interpreter - freethreaded - Python {{ShortVersion}} ({{Bitness}}, freethreaded) - Launches the !(loc.ProductName) freethreaded interpreter. - https://www.python.org/ - diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs deleted file mode 100644 index fdbcaea38eb3175..000000000000000 --- a/Tools/msi/freethreaded/freethreaded_files.wxs +++ /dev/null @@ -1,187 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/freethreaded/freethreaded_pdb.wixproj b/Tools/msi/freethreaded/freethreaded_pdb.wixproj deleted file mode 100644 index 789a4f55ae51918..000000000000000 --- a/Tools/msi/freethreaded/freethreaded_pdb.wixproj +++ /dev/null @@ -1,20 +0,0 @@ - - - - {E98E7539-64E7-4DCE-AACD-01E3ADE40EFD} - 2.0 - freethreaded_pdb - Package - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/freethreaded/freethreaded_pdb.wxs b/Tools/msi/freethreaded/freethreaded_pdb.wxs deleted file mode 100644 index 302ac416fe92753..000000000000000 --- a/Tools/msi/freethreaded/freethreaded_pdb.wxs +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/Tools/msi/generate_md5.py b/Tools/msi/generate_md5.py deleted file mode 100644 index 9e4c14731bd11db..000000000000000 --- a/Tools/msi/generate_md5.py +++ /dev/null @@ -1,27 +0,0 @@ -import hashlib -import os -import sys - -def main(): - filenames, hashes, sizes = [], [], [] - - for file in sys.argv[1:]: - if not os.path.isfile(file): - continue - - with open(file, 'rb') as f: - data = f.read() - md5 = hashlib.md5() - md5.update(data) - filenames.append(os.path.split(file)[1]) - hashes.append(md5.hexdigest()) - sizes.append(str(len(data))) - - print('{:40s} {:<32s} {:<9s}'.format('File', 'MD5', 'Size')) - for f, h, s in zip(filenames, hashes, sizes): - print('{:40s} {:>32s} {:>9s}'.format(f, h, s)) - - - -if __name__ == "__main__": - sys.exit(int(main() or 0)) diff --git a/Tools/msi/get_externals.bat b/Tools/msi/get_externals.bat deleted file mode 100644 index c7c7e9f470bc250..000000000000000 --- a/Tools/msi/get_externals.bat +++ /dev/null @@ -1,91 +0,0 @@ -@echo off -setlocal -rem Simple script to fetch source for external libraries - -set HERE=%~dp0 -if "%PCBUILD%"=="" (set PCBUILD=%HERE%..\..\PCbuild\) -if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%HERE%..\..\externals\windows-installer) -if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\..\nuget.exe) -if "%NUGET_URL%"=="" (set NUGET_URL=https://dist.nuget.org/win-x86-commandline/latest/nuget.exe) - -set DO_FETCH=true -set DO_CLEAN=false - -:CheckOpts -if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts -if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts -if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts -if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts -if "%~1"=="--clean-only" (set DO_FETCH=false) & goto clean -if "x%~1" NEQ "x" goto usage - -if "%DO_CLEAN%"=="false" goto fetch -:clean -echo.Cleaning up external libraries. -if exist "%EXTERNALS_DIR%" ( - rem Sometimes this fails the first time; try it twice - rmdir /s /q "%EXTERNALS_DIR%" || rmdir /s /q "%EXTERNALS_DIR%" -) - -if "%DO_FETCH%"=="false" goto end -:fetch - -if "%ORG%"=="" (set ORG=python) - -call "%PCBUILD%\find_python.bat" "%PYTHON%" - -echo.Fetching external libraries... - -set libraries= - -for %%e in (%libraries%) do ( - if exist "%EXTERNALS_DIR%\%%e" ( - echo.%%e already exists, skipping. - ) else ( - echo.Fetching %%e... - %PYTHON% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e - ) -) - -echo.Fetching external tools... - -set binaries= -rem We always use whatever's latest in the repo for these -set binaries=%binaries% binutils -set binaries=%binaries% gpg -set binaries=%binaries% htmlhelp -set binaries=%binaries% nuget -set binaries=%binaries% redist-1 -set binaries=%binaries% wix-314 - -for %%b in (%binaries%) do ( - if exist "%EXTERNALS_DIR%\%%b" ( - echo.%%b already exists, skipping. - ) else ( - echo.Fetching %%b... - %PYTHON% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b - ) -) - -echo Finished. -goto end - -:usage -echo.Valid options: -c, --clean, --clean-only, --organization, --python, -echo.--no-tkinter, --no-openssl -echo. -echo.Pull all sources and binaries necessary for compiling optional extension -echo.modules that rely on external libraries. -echo. -echo.The --organization option determines which github organization to download -echo.from, the --python option determines which Python 3.6+ interpreter to use -echo.with PCbuild\get_external.py. -echo. -echo.Use the -c or --clean option to remove the entire externals directory. -echo. -echo.Use the --clean-only option to do the same cleaning, without pulling in -echo.anything new. -echo. -exit /b -1 - -:end diff --git a/Tools/msi/launcher/launcher.wixproj b/Tools/msi/launcher/launcher.wixproj deleted file mode 100644 index de770bdd3006c22..000000000000000 --- a/Tools/msi/launcher/launcher.wixproj +++ /dev/null @@ -1,46 +0,0 @@ - - - - {921CF0E6-AEBC-4376-BA1D-CD46EBFE6DA5} - 2.0 - launcher - Package - UpgradeCode=1B68A0EC-4DD3-5134-840E-73854B0863F1;SuppressUpgradeTable=1;$(DefineConstants) - true - ICE80 - <_Rebuild>Build - - - - - - - - - - - - - - <_Rebuild>Rebuild - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/launcher/launcher.wxs b/Tools/msi/launcher/launcher.wxs deleted file mode 100644 index 49f1f7b8c1762ee..000000000000000 --- a/Tools/msi/launcher/launcher.wxs +++ /dev/null @@ -1,75 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - NOT Installed AND NOT ALLUSERS=1 - NOT Installed AND ALLUSERS=1 - - - UPGRADE or REMOVE_350_LAUNCHER or REMOVE_360A1_LAUNCHER or UPGRADE_3_11_0 or UPGRADE_3_11_1 - - UPGRADE or REMOVE_350_LAUNCHER or REMOVE_360A1_LAUNCHER - - - - - Installed OR NOT DOWNGRADE OR UPGRADE_3_11_0 OR UPGRADE_3_11_1 - - Installed OR NOT DOWNGRADE - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/launcher/launcher_en-US.wxl b/Tools/msi/launcher/launcher_en-US.wxl deleted file mode 100644 index a7e3827c528bb07..000000000000000 --- a/Tools/msi/launcher/launcher_en-US.wxl +++ /dev/null @@ -1,17 +0,0 @@ - - - 1033 - en-us - Python Launcher - Python Launcher - Python Launcher - Python Software Foundation - A newer version of the Python launcher is already installed. - The TARGETDIR variable must be provided when invoking this installer. - Python File - Python File (no console) - Compiled Python File - Python Extension Module - Python Zip Application File - Python Zip Application File (no console) - diff --git a/Tools/msi/launcher/launcher_files.wxs b/Tools/msi/launcher/launcher_files.wxs deleted file mode 100644 index d9a230e2d35ace2..000000000000000 --- a/Tools/msi/launcher/launcher_files.wxs +++ /dev/null @@ -1,44 +0,0 @@ - - - - - - - - - - - - - - NOT ALLUSERS=1 - - - - - - ALLUSERS=1 - - - - - VersionNT64 AND NOT ARM64_SHELLEXT - - - - - - NOT VersionNT64 AND NOT ARM64_SHELLEXT - - - - - - ARM64_SHELLEXT - - - - - - - diff --git a/Tools/msi/launcher/launcher_reg.wxs b/Tools/msi/launcher/launcher_reg.wxs deleted file mode 100644 index e8d9d24d43fa8ce..000000000000000 --- a/Tools/msi/launcher/launcher_reg.wxs +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/lib/lib.wixproj b/Tools/msi/lib/lib.wixproj deleted file mode 100644 index 3ea46dd40ea4ce9..000000000000000 --- a/Tools/msi/lib/lib.wixproj +++ /dev/null @@ -1,35 +0,0 @@ - - - - {11367E76-3337-4602-8F1E-77DB4F370D7E} - 2.0 - lib - Package - - - - - - - - - - - - - $(PySourcePath)Lib - !(bindpath.src)Lib\ - $(PySourcePath)Lib - Lib\ - lib_py - true - - - - - diff --git a/Tools/msi/lib/lib.wxs b/Tools/msi/lib/lib.wxs deleted file mode 100644 index e417e31b33783ed..000000000000000 --- a/Tools/msi/lib/lib.wxs +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/lib/lib_d.wixproj b/Tools/msi/lib/lib_d.wixproj deleted file mode 100644 index 587a82c1929da7d..000000000000000 --- a/Tools/msi/lib/lib_d.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {6C443CD3-8258-4335-BA03-49DA9C34CE4D} - 2.0 - lib_d - Package - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/lib/lib_d.wxs b/Tools/msi/lib/lib_d.wxs deleted file mode 100644 index 7968d179743a009..000000000000000 --- a/Tools/msi/lib/lib_d.wxs +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/Tools/msi/lib/lib_en-US.wxl b/Tools/msi/lib/lib_en-US.wxl deleted file mode 100644 index 305bcc73afc8082..000000000000000 --- a/Tools/msi/lib/lib_en-US.wxl +++ /dev/null @@ -1,5 +0,0 @@ - - - Standard Library - lib - diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs deleted file mode 100644 index 4d44299f7839099..000000000000000 --- a/Tools/msi/lib/lib_files.wxs +++ /dev/null @@ -1,128 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/lib/lib_pdb.wixproj b/Tools/msi/lib/lib_pdb.wixproj deleted file mode 100644 index db1b5bb316cfead..000000000000000 --- a/Tools/msi/lib/lib_pdb.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {5E0BCE93-D1AC-4591-BBCB-3A2BE5A4B3D1} - 2.0 - lib_pdb - Package - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/lib/lib_pdb.wxs b/Tools/msi/lib/lib_pdb.wxs deleted file mode 100644 index 7918893a9a25050..000000000000000 --- a/Tools/msi/lib/lib_pdb.wxs +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/Tools/msi/make_appx.ps1 b/Tools/msi/make_appx.ps1 deleted file mode 100644 index e32bd76a37f135a..000000000000000 --- a/Tools/msi/make_appx.ps1 +++ /dev/null @@ -1,71 +0,0 @@ -<# -.Synopsis - Compiles and signs an APPX package -.Description - Given the file listing, ensures all the contents are signed - and builds and signs the final package. -.Parameter mapfile - The location on disk of the text mapping file. -.Parameter msix - The path and name to store the APPX/MSIX. -.Parameter sign - When set, signs the APPX/MSIX. Packages to be published to - the store should not be signed. -.Parameter description - Description to embed in the signature (optional). -.Parameter certname - The name of the certificate to sign with (optional). -.Parameter certsha1 - The SHA1 hash of the certificate to sign with (optional). -#> -param( - [Parameter(Mandatory=$true)][string]$layout, - [Parameter(Mandatory=$true)][string]$msix, - [switch]$sign, - [string]$description, - [string]$certname, - [string]$certsha1, - [string]$certfile -) - -$tools = $script:MyInvocation.MyCommand.Path | Split-Path -parent; -Import-Module $tools\sdktools.psm1 -WarningAction SilentlyContinue -Force - -Set-Alias makeappx (Find-Tool "makeappx.exe") -Scope Script -Set-Alias makepri (Find-Tool "makepri.exe") -Scope Script - -$msixdir = Split-Path $msix -Parent -if ($msixdir) { - $msixdir = (mkdir -Force $msixdir).FullName -} else { - $msixdir = Get-Location -} -$msix = Join-Path $msixdir (Split-Path $msix -Leaf) - -pushd $layout -try { - if (Test-Path resources.pri) { - del resources.pri - } - $name = ([xml](gc AppxManifest.xml)).Package.Identity.Name - makepri new /pr . /mn AppxManifest.xml /in $name /cf _resources.xml /of _resources.pri /mf appx /o - if (-not $? -or -not (Test-Path _resources.map.txt)) { - throw "makepri step failed" - } - $lines = gc _resources.map.txt - $lines | ?{ -not ($_ -match '"_resources[\w\.]+?"') } | Out-File _resources.map.txt -Encoding utf8 - makeappx pack /f _resources.map.txt /m AppxManifest.xml /o /p $msix - if (-not $?) { - throw "makeappx step failed" - } -} finally { - popd -} - -if ($sign) { - Sign-File -certname $certname -certsha1 $certsha1 -certfile $certfile -description $description -files $msix - - if (-not $?) { - throw "Package signing failed" - } -} diff --git a/Tools/msi/make_cat.ps1 b/Tools/msi/make_cat.ps1 deleted file mode 100644 index 9ea3ddd495719e6..000000000000000 --- a/Tools/msi/make_cat.ps1 +++ /dev/null @@ -1,45 +0,0 @@ -<# -.Synopsis - Compiles and signs a catalog file. -.Description - Given the CDF definition file, builds and signs a catalog. -.Parameter catalog - The path to the catalog definition file to compile and - sign. It is assumed that the .cat file will be the same - name with a new extension. -.Parameter outfile - The path to move the built .cat file to (optional). -.Parameter description - The description to add to the signature (optional). -.Parameter certname - The name of the certificate to sign with (optional). -.Parameter certsha1 - The SHA1 hash of the certificate to sign with (optional). -#> -param( - [Parameter(Mandatory=$true)][string]$catalog, - [string]$outfile, - [switch]$sign, - [string]$description, - [string]$certname, - [string]$certsha1, - [string]$certfile -) - -$tools = $script:MyInvocation.MyCommand.Path | Split-Path -parent; -Import-Module $tools\sdktools.psm1 -WarningAction SilentlyContinue -Force - -Set-Alias MakeCat (Find-Tool "makecat.exe") -Scope Script - -MakeCat $catalog -if (-not $?) { - throw "Catalog compilation failed" -} -if ($sign) { - Sign-File -certname $certname -certsha1 $certsha1 -certfile $certfile -description $description -files @($catalog -replace 'cdf$', 'cat') -} - -if ($outfile) { - Split-Path -Parent $outfile | ?{ $_ } | %{ mkdir -Force $_; } - Move-Item ($catalog -replace 'cdf$', 'cat') $outfile -} diff --git a/Tools/msi/make_zip.proj b/Tools/msi/make_zip.proj deleted file mode 100644 index 125a434e51f4147..000000000000000 --- a/Tools/msi/make_zip.proj +++ /dev/null @@ -1,40 +0,0 @@ - - - - {10487945-15D1-4092-A214-338395C4116B} - python - - false - - - - - - false - python-$(PythonVersion)-embed-$(ArchName) - .zip - $(OutputPath)\$(TargetName)$(TargetExt) - rmdir /q/s "$(IntermediateOutputPath)\zip_$(ArchName)" - "$(PythonExe)" "$(PySourcePath)PC\layout" - $(Arguments) -b "$(BuildPath.TrimEnd(`\`))" -s "$(PySourcePath.TrimEnd(`\`))" - $(Arguments) -t "$(IntermediateOutputPath)\zip_$(ArchName)" - $(Arguments) --zip "$(TargetPath)" - $(Arguments) --precompile --zip-lib --include-underpth --include-stable --flat-dlls - $(Environment)%0D%0Aset PYTHONPATH=$(PySourcePath)Lib - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/msi.props b/Tools/msi/msi.props deleted file mode 100644 index 097af60715448f1..000000000000000 --- a/Tools/msi/msi.props +++ /dev/null @@ -1,234 +0,0 @@ - - - - $(OutputName) - false - false - $(SuppressIces);ICE03;ICE57;ICE61 - 1026 - false - true - Release - x86 - perUser - <_MakeCatCommand Condition="'$(_MakeCatCommand)' == ''">makecat - true - - - - - - - - - $(ComputerName)/$(ArchName)/ - $(ReleaseUri)/ - - - - - - - - WixUtilExtension - WixUtilExtension - - - - - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\msi_$(OutputName) - $(IntermediateOutputPath)_$(OutputSuffix) - $(BuildPath) - $(OutputPath)\ - $(OutputPath) - true - $(ExternalsDir)\windows-installer\redist-1\$(Platform) - $([System.IO.Path]::GetFullPath($(CRTRedist))) - $(tcltkDir)lib - python$(MajorVersionNumber)$(MinorVersionNumber)$(MicroVersionNumber)$(ReleaseLevelName).chm - - $(MajorVersionNumber).$(MinorVersionNumber).$(Field3Value).0 - - - - false - - - - $([System.Math]::Floor($([System.DateTime]::Now.Subtract($([System.DateTime]::new(2001, 1, 1))).TotalDays))) - $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)dev$(RevisionNumber) - $(MajorVersionNumber).$(MinorVersionNumber).$(RevisionNumber).0 - - - - 32-bit - 64-bit - ARM64 - 32bit - 64bit - ARM64 - - $(DefineConstants); - Version=$(InstallerVersion); - ShortVersion=$(MajorVersionNumber).$(MinorVersionNumber); - LongVersion=$(PythonVersion); - MajorVersionNumber=$(MajorVersionNumber); - MinorVersionNumber=$(MinorVersionNumber); - UpgradeMinimumVersion=$(MajorVersionNumber).$(MinorVersionNumber).0.0; - NextMajorVersionNumber=$(MajorVersionNumber).$([msbuild]::Add($(MinorVersionNumber), 1)).0.0; - Bitness=$(Bitness); - PlatformArchitecture=$(PlatformArchitecture); - PyDebugExt=$(PyDebugExt); - PyArchExt=$(PyArchExt); - PyTestExt=$(PyTestExt); - PydTag=$(PydTag); - FreethreadedPydTag=$(FreethreadedPydTag); - OptionalFeatureName=$(OutputName); - ssltag=$(OpenSSLDLLSuffix); - Suffix32=$(PyArchExt); - IncludeFreethreaded=$(IncludeFreethreaded); - - - $(DefineConstants);CRTRedist=$(CRTRedist); - - - - - - generated_filelist - - - false - - - false - - - - - - - - - build - - - build - - - build - - - build_t - - - build_t - - - build_t - - - - src - - - tcltk - - - redist - - - - build32 - - - build64 - - - buildarm64 - - - build32t - - - build64t - - - buildarm64t - - - - - - - - - <_Uuid Include="CoreUpgradeCode"> - upgradecode - - <_Uuid Include="UpgradeCode"> - upgradecode/$(OutputName) - - <_Uuid Include="InstallDirectoryGuidSeed"> - installdirectoryseed - - <_Uuid Include="PythonExeComponentGuid"> - python.exe - - <_Uuid Include="PythonwExeComponentGuid"> - pythonw.exe - - <_Uuid Include="CommonPythonRegComponentGuid"> - registry - - <_Uuid Include="PythonRegComponentGuid"> - registry/$(OutputName) - - <_Uuid Include="FreethreadedPythonExeComponentGuid" Condition="$(IncludeFreethreaded)"> - freethreaded/python.exe - - <_Uuid Include="FreethreadedPythonwExeComponentGuid" Condition="$(IncludeFreethreaded)"> - freethreaded/pythonw.exe - - - - - <_Uuids>@(_Uuid->'("%(Identity)", "$(MajorVersionNumber).$(MinorVersionNumber)/%(Uri)")',',') - <_GenerateCommand>import uuid; print('\n'.join('{}={}'.format(i, uuid.uuid5(uuid.UUID('c8d9733e-a70c-43ff-ab0c-e26456f11083'), '$(ReleaseUri.Replace(`{arch}`, `$(ArchName)`))' + j)) for i,j in [$(_Uuids.Replace(`"`,`'`))])) - - - - - - - - - - $(DefineConstants);@(_UuidValue,';'); - - - diff --git a/Tools/msi/msi.targets b/Tools/msi/msi.targets deleted file mode 100644 index 99d1de4d9d83f58..000000000000000 --- a/Tools/msi/msi.targets +++ /dev/null @@ -1,96 +0,0 @@ - - - - - - <_FileListTarget>$(IntermediateOutputPath)$(MSBuildProjectName).g.csv - <_InstallFilesTarget>$(IntermediateOutputPath)$(MSBuildProjectName).g.wxs - - - - - <_Source>%(Source)$([msbuild]::MakeRelative(%(SourceBase), %(FullPath))) - <_Target>%(Target_)$([msbuild]::MakeRelative(%(TargetBase), %(FullPath))) - - - <_CatalogFiles Include="@(InstallFiles)" Condition="%(InstallFiles.IncludeInCat) and ''!=$([System.IO.File]::ReadAllText(%(InstallFiles.FullPath)))" /> - - - - - - - - - - - - - - <_CatFileSourceTarget>$(IntermediateOutputPath)$(MSBuildProjectName).cdf - <_CatFileTarget>$(IntermediateOutputPath)python_$(MSBuildProjectName).cat - <_CatFile>[CatalogHeader] -Name=$([System.IO.Path]::GetFileName($(_CatFileTarget))) -ResultDir=$([System.IO.Path]::GetDirectoryName($(_CatFileTarget))) -PublicVersion=1 -CatalogVersion=2 -HashAlgorithms=SHA256 -PageHashes=false -EncodingType= - -[CatalogFiles] -@(_CatalogFiles->'<HASH>%(Filename)%(Extension)=%(FullPath)',' -') - - - - - - - - - - - - - - - <_Content>$([System.IO.File]::ReadAllText(%(WxlTemplate.FullPath)).Replace(`{{ShortVersion}}`, `$(MajorVersionNumber).$(MinorVersionNumber)$(PyTestExt)`).Replace(`{{LongVersion}}`, `$(PythonVersion)$(PyTestExt)`).Replace(`{{Bitness}}`, `$(Bitness)`)) - <_ExistingContent Condition="Exists('$(IntermediateOutputPath)%(WxlTemplate.Filename).wxl')">$([System.IO.File]::ReadAllText($(IntermediateOutputPath)%(WxlTemplate.Filename).wxl)) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/path/path.wixproj b/Tools/msi/path/path.wixproj deleted file mode 100644 index 2792e14666d3845..000000000000000 --- a/Tools/msi/path/path.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {91C99298-8E2E-4422-A5AF-CC4FFF9A58D3} - 2.0 - path - Package - ICE71 - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/path/path.wxs b/Tools/msi/path/path.wxs deleted file mode 100644 index 32854392e8bee3f..000000000000000 --- a/Tools/msi/path/path.wxs +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - NOT ALLUSERS=1 - - - - - - - - - - - - ALLUSERS=1 - - - - - - - - - - - - - - - - diff --git a/Tools/msi/path/path_en-US.wxl b/Tools/msi/path/path_en-US.wxl deleted file mode 100644 index 33a7886fe2d6e20..000000000000000 --- a/Tools/msi/path/path_en-US.wxl +++ /dev/null @@ -1,6 +0,0 @@ - - - Add to Path - Path - No !(loc.ProductName) installation was detected. - diff --git a/Tools/msi/pip/pip.wixproj b/Tools/msi/pip/pip.wixproj deleted file mode 100644 index 718c02c032b73b3..000000000000000 --- a/Tools/msi/pip/pip.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {91C99298-8E2E-4422-A5AF-CC4FFF9A58D3} - 2.0 - pip - Package - ICE71 - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/pip/pip.wxs b/Tools/msi/pip/pip.wxs deleted file mode 100644 index 627c4710a9fdfaa..000000000000000 --- a/Tools/msi/pip/pip.wxs +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - PYTHON_EXE - - - - - - - - - - - - - - - - - (&DefaultFeature=3) AND NOT (!DefaultFeature=3) - (&DefaultFeature=2) AND (!DefaultFeature=3) - - UpdatePip - - - - diff --git a/Tools/msi/pip/pip_en-US.wxl b/Tools/msi/pip/pip_en-US.wxl deleted file mode 100644 index cd0d9edf3a6b6fd..000000000000000 --- a/Tools/msi/pip/pip_en-US.wxl +++ /dev/null @@ -1,6 +0,0 @@ - - - pip Bootstrap - pip - No !(loc.ProductName) installation was detected. - diff --git a/Tools/msi/purge.py b/Tools/msi/purge.py deleted file mode 100644 index 4a13d368d7c744e..000000000000000 --- a/Tools/msi/purge.py +++ /dev/null @@ -1,76 +0,0 @@ -# Purges the Fastly cache for Windows download files -# -# Usage: -# py -3 purge.py 3.5.1rc1 -# - -__author__ = 'Steve Dower ' -__version__ = '1.0.0' - -import re -import sys - -from urllib.request import Request, urlopen - -VERSION_RE = re.compile(r'(\d+\.\d+\.\d+)([A-Za-z_]+\d+)?$') - -try: - m = VERSION_RE.match(sys.argv[1]) - if not m: - print('Invalid version:', sys.argv[1]) - print('Expected something like "3.5.1rc1"') - sys.exit(1) -except LookupError: - print('Missing version argument. Expected something like "3.5.1rc1"') - sys.exit(1) - -URL = "https://www.python.org/ftp/python/{}/".format(m.group(1)) -REL = m.group(2) or '' - -FILES = [ - "core.msi", - "core_d.msi", - "core_pdb.msi", - "dev.msi", - "dev_d.msi", - "doc.msi", - "exe.msi", - "exe_d.msi", - "exe_pdb.msi", - "launcher.msi", - "lib.msi", - "lib_d.msi", - "lib_pdb.msi", - "path.msi", - "pip.msi", - "tcltk.msi", - "tcltk_d.msi", - "tcltk_pdb.msi", - "test.msi", - "test_d.msi", - "test_pdb.msi", - "tools.msi", - "ucrt.msi", -] -PATHS = [ - "python-{}.exe".format(m.group(0)), - "python-{}-webinstall.exe".format(m.group(0)), - "python-{}-amd64.exe".format(m.group(0)), - "python-{}-amd64-webinstall.exe".format(m.group(0)), - "python-{}-arm64.exe".format(m.group(0)), - "python-{}-arm64-webinstall.exe".format(m.group(0)), - "python-{}-embed-amd64.zip".format(m.group(0)), - "python-{}-embed-win32.zip".format(m.group(0)), - "python-{}-embed-arm64.zip".format(m.group(0)), - *["win32{}/{}".format(REL, f) for f in FILES], - *["amd64{}/{}".format(REL, f) for f in FILES], - *["arm64{}/{}".format(REL, f) for f in FILES], -] -PATHS = PATHS + [p + ".asc" for p in PATHS] - -print('Purged:') -for n in PATHS: - u = URL + n - with urlopen(Request(u, method='PURGE', headers={'Fastly-Soft-Purge': 1})) as r: - r.read() - print(' ', u) diff --git a/Tools/msi/sdktools.psm1 b/Tools/msi/sdktools.psm1 deleted file mode 100644 index c5973f9abc6abf0..000000000000000 --- a/Tools/msi/sdktools.psm1 +++ /dev/null @@ -1,48 +0,0 @@ -function Find-Tool { - param([string]$toolname) - - $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 - $tool = (gci -r "$kitroot\Bin\*\x64\$toolname" | sort FullName -Desc | select -First 1) - if (-not $tool) { - throw "$toolname is not available" - } - Write-Host "Found $toolname at $($tool.FullName)" - return $tool.FullName -} - -Set-Alias SignTool (Find-Tool "signtool.exe") -Scope Script - -function Sign-File { - param([string]$certname, [string]$certsha1, [string]$certfile, [string]$description, [string[]]$files) - - if (-not $description) { - $description = $env:SigningDescription; - if (-not $description) { - $description = "Python"; - } - } - if (-not $certsha1) { - $certsha1 = $env:SigningCertificateSha1; - } - if (-not $certname) { - $certname = $env:SigningCertificate; - } - if (-not $certfile) { - $certfile = $env:SigningCertificateFile; - } - - if (-not ($certsha1 -or $certname -or $certfile)) { - throw "No signing certificate specified" - } - - foreach ($a in $files) { - if ($certsha1) { - SignTool sign /sha1 $certsha1 /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d $description $a - } elseif ($certname) { - SignTool sign /a /n $certname /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d $description $a - } elseif ($certfile) { - SignTool sign /f $certfile /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d $description $a - } - } -} - diff --git a/Tools/msi/sign_build.ps1 b/Tools/msi/sign_build.ps1 deleted file mode 100644 index d3f750454f522b0..000000000000000 --- a/Tools/msi/sign_build.ps1 +++ /dev/null @@ -1,34 +0,0 @@ -<# -.Synopsis - Recursively signs the contents of a directory. -.Description - Given the file patterns, code signs the contents. -.Parameter root - The root directory to sign. -.Parameter patterns - The file patterns to sign -.Parameter description - The description to add to the signature (optional). -.Parameter certname - The name of the certificate to sign with (optional). -.Parameter certsha1 - The SHA1 hash of the certificate to sign with (optional). -#> -param( - [Parameter(Mandatory=$true)][string]$root, - [string[]]$patterns=@("*.exe", "*.dll", "*.pyd", "*.cat"), - [string]$description, - [string]$certname, - [string]$certsha1, - [string]$certfile -) - -$tools = $script:MyInvocation.MyCommand.Path | Split-Path -parent; -Import-Module $tools\sdktools.psm1 -WarningAction SilentlyContinue -Force - -pushd $root -try { - Sign-File -certname $certname -certsha1 $certsha1 -certfile $certfile -description $description -files (gci -r $patterns) -} finally { - popd -} \ No newline at end of file diff --git a/Tools/msi/tcltk/tcltk.wixproj b/Tools/msi/tcltk/tcltk.wixproj deleted file mode 100644 index c8b7ab77c4dc6b7..000000000000000 --- a/Tools/msi/tcltk/tcltk.wixproj +++ /dev/null @@ -1,42 +0,0 @@ - - - - {DB350600-186C-4E52-BA98-26A7CECB067F} - 2.0 - tcltk - Package - - - - ICE43 - - - - - - - - - - - - - $(TclTkLibraryDir) - !(bindpath.tcltk) - $(TclTkLibraryDir) - tcl\ - tcltk_lib - - - - $(PySourcePath) - !(bindpath.src) - $(PySourcePath) - - tkinter_lib - - - - - diff --git a/Tools/msi/tcltk/tcltk.wxs b/Tools/msi/tcltk/tcltk.wxs deleted file mode 100644 index bad56d55cce3723..000000000000000 --- a/Tools/msi/tcltk/tcltk.wxs +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - PYTHON_EXE and PYTHONW_EXE - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/tcltk/tcltk_d.wixproj b/Tools/msi/tcltk/tcltk_d.wixproj deleted file mode 100644 index 3266190da0c9a6a..000000000000000 --- a/Tools/msi/tcltk/tcltk_d.wixproj +++ /dev/null @@ -1,28 +0,0 @@ - - - - {EDA1FA5A-E2AA-4EAF-B49B-87D981CD0F16} - 2.0 - tcltk_d - Package - - - - - - - - - - - - $(tcltkDir) - !(bindpath.tcltk) - $(tcltkDir)bin - DLLs\ - tcltk_dlls_d - - - - - \ No newline at end of file diff --git a/Tools/msi/tcltk/tcltk_d.wxs b/Tools/msi/tcltk/tcltk_d.wxs deleted file mode 100644 index 38aacd929995efa..000000000000000 --- a/Tools/msi/tcltk/tcltk_d.wxs +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/Tools/msi/tcltk/tcltk_en-US.wxl_template b/Tools/msi/tcltk/tcltk_en-US.wxl_template deleted file mode 100644 index f40fd6239deaa36..000000000000000 --- a/Tools/msi/tcltk/tcltk_en-US.wxl_template +++ /dev/null @@ -1,12 +0,0 @@ - - - Tcl/Tk Support - tcltk - No !(loc.ProductName) installation was detected. - IDLE (Python {{ShortVersion}} {{Bitness}}) - Launches IDLE, the interactive environment for !(loc.ProductName). - Python {{ShortVersion}} Module Docs ({{Bitness}}) - Start the !(loc.ProductName) documentation server. - &Edit with IDLE - Edit with IDLE {{ShortVersion}} ({{Bitness}}) - diff --git a/Tools/msi/tcltk/tcltk_files.wxs b/Tools/msi/tcltk/tcltk_files.wxs deleted file mode 100644 index 7c7784741d9178d..000000000000000 --- a/Tools/msi/tcltk/tcltk_files.wxs +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/tcltk/tcltk_pdb.wixproj b/Tools/msi/tcltk/tcltk_pdb.wixproj deleted file mode 100644 index 3370798a0559dba..000000000000000 --- a/Tools/msi/tcltk/tcltk_pdb.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {02053AFA-1831-499A-B3EA-D8B223D3C40D} - 2.0 - tcltk_pdb - Package - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/tcltk/tcltk_pdb.wxs b/Tools/msi/tcltk/tcltk_pdb.wxs deleted file mode 100644 index 55f209162553eb0..000000000000000 --- a/Tools/msi/tcltk/tcltk_pdb.wxs +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/Tools/msi/tcltk/tcltk_reg.wxs b/Tools/msi/tcltk/tcltk_reg.wxs deleted file mode 100644 index 2778bcc80669b6d..000000000000000 --- a/Tools/msi/tcltk/tcltk_reg.wxs +++ /dev/null @@ -1,48 +0,0 @@ - - - - - - - VersionNT > 600 - - - - - VersionNT > 600 - - - - - - VersionNT > 600 - - - - - - - VersionNT > 600 - - - - - - - - VersionNT = 600 - - - - - - - VersionNT = 600 - - - - - - - - diff --git a/Tools/msi/test/test.wixproj b/Tools/msi/test/test.wixproj deleted file mode 100644 index 8347e3f1d3db65b..000000000000000 --- a/Tools/msi/test/test.wixproj +++ /dev/null @@ -1,29 +0,0 @@ - - - - {DE0B7CC2-4358-4131-B3F4-C31C7F2CD468} - 2.0 - test - Package - - - - - - - - - - - - $(PySourcePath) - !(bindpath.src) - $(PySourcePath) - - test_py - - - - - \ No newline at end of file diff --git a/Tools/msi/test/test.wxs b/Tools/msi/test/test.wxs deleted file mode 100644 index 9e497e73a94f5b5..000000000000000 --- a/Tools/msi/test/test.wxs +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/test/test_d.wixproj b/Tools/msi/test/test_d.wixproj deleted file mode 100644 index 33b04be1fe60d63..000000000000000 --- a/Tools/msi/test/test_d.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {41F5AE8D-24CD-4D03-BE75-AA6F7FAB4097} - 2.0 - test_d - Package - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/test/test_d.wxs b/Tools/msi/test/test_d.wxs deleted file mode 100644 index 9b26ff6a4bf4cc3..000000000000000 --- a/Tools/msi/test/test_d.wxs +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/Tools/msi/test/test_en-US.wxl b/Tools/msi/test/test_en-US.wxl deleted file mode 100644 index e615c7a6b7f7062..000000000000000 --- a/Tools/msi/test/test_en-US.wxl +++ /dev/null @@ -1,7 +0,0 @@ - - - Test Suite - test - !(loc.FullProductName) native libtest - !(loc.ProductName) Native Test Modules - diff --git a/Tools/msi/test/test_files.wxs b/Tools/msi/test/test_files.wxs deleted file mode 100644 index 6862a5899db2bac..000000000000000 --- a/Tools/msi/test/test_files.wxs +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/test/test_pdb.wixproj b/Tools/msi/test/test_pdb.wixproj deleted file mode 100644 index 965f0edd9b49924..000000000000000 --- a/Tools/msi/test/test_pdb.wixproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - {7CF48ADD-CFAA-499F-9A05-BA18440A3344} - 2.0 - test_pdb - Package - - - - - - - - - - - - \ No newline at end of file diff --git a/Tools/msi/test/test_pdb.wxs b/Tools/msi/test/test_pdb.wxs deleted file mode 100644 index 5a2ef3668400756..000000000000000 --- a/Tools/msi/test/test_pdb.wxs +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/Tools/msi/testrelease.bat b/Tools/msi/testrelease.bat deleted file mode 100644 index db98f690151196c..000000000000000 --- a/Tools/msi/testrelease.bat +++ /dev/null @@ -1,115 +0,0 @@ -@setlocal enableextensions -@echo off - -set D=%~dp0 -set PCBUILD=%D%..\..\PCbuild\ - -set TARGETDIR=%TEMP% -set TESTX86= -set TESTX64= -set TESTALLUSER= -set TESTPERUSER= - -:CheckOpts -if "%1" EQU "-h" goto Help -if "%1" EQU "-x86" (set TESTX86=1) && shift && goto CheckOpts -if "%1" EQU "-x64" (set TESTX64=1) && shift && goto CheckOpts -if "%1" EQU "-t" (set TARGETDIR=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--target" (set TARGETDIR=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-a" (set TESTALLUSER=1) && shift && goto CheckOpts -if "%1" EQU "--alluser" (set TESTALLUSER=1) && shift && goto CheckOpts -if "%1" EQU "-p" (set TESTPERUSER=1) && shift && goto CheckOpts -if "%1" EQU "--peruser" (set TESTPERUSER=1) && shift && goto CheckOpts - -if not defined TESTX86 if not defined TESTX64 (set TESTX86=1) && (set TESTX64=1) -if not defined TESTALLUSER if not defined TESTPERUSER (set TESTALLUSER=1) && (set TESTPERUSER=1) - - -if defined TESTX86 ( - for %%f in ("%PCBUILD%win32\en-us\*.exe") do ( - if defined TESTALLUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-alluser" "InstallAllUsers=1 CompileAll=1" - if errorlevel 1 exit /B - if defined TESTPERUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-peruser" "InstallAllUsers=0 CompileAll=0" - if errorlevel 1 exit /B - ) -) - -if defined TESTX64 ( - for %%f in ("%PCBUILD%amd64\en-us\*.exe") do ( - if defined TESTALLUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-alluser" "InstallAllUsers=1 CompileAll=1" - if errorlevel 1 exit /B - if defined TESTPERUSER call :test "%%~ff" "%TARGETDIR%\%%~nf-peruser" "InstallAllUsers=0 CompileAll=0" - if errorlevel 1 exit /B - ) -) - -exit /B 0 - -:test -@setlocal -@echo on - -@if not exist "%~1" exit /B 1 - -@set EXE=%~1 -@if not "%EXE:embed=%"=="%EXE%" exit /B 0 - -@set EXITCODE=0 -@echo Installing %1 into %2 -"%~1" /passive /log "%~2\install\log.txt" TargetDir="%~2\Python" Include_debug=1 Include_symbols=1 %~3 - -@if not errorlevel 1 ( - @echo Printing version - "%~2\Python\python.exe" -c "import sys; print(sys.version)" > "%~2\version.txt" 2>&1 -) - -@if not errorlevel 1 ( - @echo Capturing Start Menu - @dir /s/b "%PROGRAMDATA%\Microsoft\Windows\Start Menu\Programs" | findstr /ic:"python" > "%~2\startmenu.txt" 2>&1 - @dir /s/b "%APPDATA%\Microsoft\Windows\Start Menu\Programs" | findstr /ic:"python" >> "%~2\startmenu.txt" 2>&1 - - @echo Capturing registry - @for /F "usebackq" %%f in (`reg query HKCR /s /f python /k`) do @( - echo %%f >> "%~2\hkcr.txt" - reg query "%%f" /s >> "%~2\hkcr.txt" 2>&1 - ) - @reg query HKCU\Software\Python /s > "%~2\hkcu.txt" 2>&1 - @reg query HKLM\Software\Python /reg:32 /s > "%~2\hklm.txt" 2>&1 - @reg query HKLM\Software\Python /reg:64 /s >> "%~2\hklm.txt" 2>&1 - cmd /k exit 0 -) - -@if not errorlevel 1 ( - @echo Installing package - "%~2\Python\python.exe" -m pip install "azure<0.10" > "%~2\pip.txt" 2>&1 - @if not errorlevel 1 ( - "%~2\Python\python.exe" -m pip uninstall -y azure python-dateutil six >> "%~2\pip.txt" 2>&1 - ) -) -@if not errorlevel 1 ( - @echo Testing Tcl/tk - "%~2\Python\python.exe" -m test -uall -v test_ttk_guionly test_tk test_idle > "%~2\tcltk.txt" 2>&1 -) - -@set EXITCODE=%ERRORLEVEL% - -@echo Result was %EXITCODE% -@echo Removing %1 -"%~1" /passive /uninstall /log "%~2\uninstall\log.txt" - -@echo off -exit /B %EXITCODE% - -:Help -echo testrelease.bat [--target TARGET] [-x86] [-x64] [--alluser] [--peruser] [-h] -echo. -echo --target (-t) Specify the target directory for installs and logs -echo -x86 Run tests for x86 installers -echo -x64 Run tests for x64 installers -echo --alluser (-a) Run tests for all-user installs (requires Administrator) -echo --peruser (-p) Run tests for per-user installs -echo -h Display this help information -echo. -echo If no test architecture is specified, all architectures will be tested. -echo If no install type is selected, all install types will be tested. -echo. diff --git a/Tools/msi/ucrt/ucrt.wixproj b/Tools/msi/ucrt/ucrt.wixproj deleted file mode 100644 index 52a7e0e672b9aff..000000000000000 --- a/Tools/msi/ucrt/ucrt.wixproj +++ /dev/null @@ -1,28 +0,0 @@ - - - - {E233091D-2EE3-49D8-A7F1-6E266CE1B6B1} - 2.0 - ucrt - Package - - - - - - - - - - - $(CRTRedist) - !(bindpath.redist)\ - $(CRTRedist)\ucrt - InstallDirectory\ - ucrt_dll - false - - - - - \ No newline at end of file diff --git a/Tools/msi/ucrt/ucrt.wxs b/Tools/msi/ucrt/ucrt.wxs deleted file mode 100644 index e9e2a9a9045115b..000000000000000 --- a/Tools/msi/ucrt/ucrt.wxs +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/Tools/msi/ucrt/ucrt_en-US.wxl b/Tools/msi/ucrt/ucrt_en-US.wxl deleted file mode 100644 index f42dd5fe44cd590..000000000000000 --- a/Tools/msi/ucrt/ucrt_en-US.wxl +++ /dev/null @@ -1,5 +0,0 @@ - - - C Runtime Library - ucrt - diff --git a/Tools/msi/uploadrelease.bat b/Tools/msi/uploadrelease.bat deleted file mode 100644 index f262489cce60857..000000000000000 --- a/Tools/msi/uploadrelease.bat +++ /dev/null @@ -1,109 +0,0 @@ -@setlocal -@echo off - -set D=%~dp0 -set PCBUILD=%D%..\..\PCbuild\ - -set HOST= -set USER= -set TARGET= -set DRYRUN=false -set NOUPLOAD= -set NOGPG= -set NOPURGE= -set NOTEST= - -:CheckOpts -if "%1" EQU "-h" goto Help -if "%1" EQU "-o" (set HOST=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--host" (set HOST=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-u" (set USER=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--user" (set USER=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "-t" (set TARGET=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--target" (set TARGET=%~2) && shift && shift && goto CheckOpts -if "%1" EQU "--dry-run" (set DRYRUN=true) && shift && goto CheckOpts -if "%1" EQU "--skip-upload" (set NOUPLOAD=true) && shift && goto CheckOpts -if "%1" EQU "--skip-gpg" (set NOGPG=true) && shift && goto CheckOpts -if "%1" EQU "--skip-purge" (set NOPURGE=true) && shift && goto CheckOpts -if "%1" EQU "--skip-test" (set NOTEST=true) && shift && goto CheckOpts -if "%1" EQU "-T" (set NOTEST=true) && shift && goto CheckOpts -if "%1" NEQ "" echo Unexpected argument "%1" & exit /B 1 - -if not defined PLINK where plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" -if not defined PLINK where /R "%ProgramFiles(x86)%\PuTTY" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" -if not defined PLINK where /R "%ProgramFiles%\PuTTY" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" -if not defined PLINK where /R "%ProgramFiles(x86)%" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" -if not defined PLINK where /R "%ProgramFiles%" plink > "%TEMP%\plink.loc" 2> nul && set /P PLINK= < "%TEMP%\plink.loc" & del "%TEMP%\plink.loc" -if not defined PLINK echo Cannot locate plink.exe & exit /B 1 -echo Found plink.exe at %PLINK% - -if not defined PSCP where pscp > "%TEMP%\pscp.loc" 2> nul && set /P pscp= < "%TEMP%\pscp.loc" & del "%TEMP%\pscp.loc" -if not defined PSCP where /R "%ProgramFiles(x86)%\PuTTY" pscp > "%TEMP%\pscp.loc" 2> nul && set /P pscp= < "%TEMP%\pscp.loc" & del "%TEMP%\pscp.loc" -if not defined PSCP where /R "%ProgramFiles%\PuTTY" pscp > "%TEMP%\pscp.loc" 2> nul && set /P pscp= < "%TEMP%\pscp.loc" & del "%TEMP%\pscp.loc" -if not defined PSCP where /R "%ProgramFiles(x86)%" pscp > "%TEMP%\pscp.loc" 2> nul && set /P pscp= < "%TEMP%\pscp.loc" & del "%TEMP%\pscp.loc" -if not defined PSCP where /R "%ProgramFiles%" pscp > "%TEMP%\pscp.loc" 2> nul && set /P pscp= < "%TEMP%\pscp.loc" & del "%TEMP%\pscp.loc" -if not defined PSCP echo Cannot locate pscp.exe & exit /B 1 -echo Found pscp.exe at %PSCP% - -if defined NOGPG ( - set GPG= - echo Skipping GPG signature generation because of --skip-gpg -) else ( - if not defined GPG where gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" - if not defined GPG where /R "%PCBUILD%..\externals\windows-installer" gpg2 > "%TEMP%\gpg.loc" 2> nul && set /P GPG= < "%TEMP%\gpg.loc" & del "%TEMP%\gpg.loc" - if not defined GPG echo Cannot locate gpg2.exe. Signatures will not be uploaded & pause - echo Found gpg2.exe at %GPG% -) - -call "%PCBUILD%find_msbuild.bat" %MSBUILD% -if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) -pushd "%D%" -if not defined NOUPLOAD ( - %MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x86 - if errorlevel 1 goto :failed - %MSBUILD% /v:m /nologo uploadrelease.proj /t:Upload /p:Platform=x64 /p:IncludeDoc=false - if errorlevel 1 goto :failed -) -if not defined NOPURGE ( - %MSBUILD% /v:m /nologo uploadrelease.proj /t:Purge -) -if not defined NOTEST ( - call :test x86 - if errorlevel 1 goto :failed - call :test x64 - if errorlevel 1 goto :failed -) -%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x86 -if errorlevel 1 goto :failed -%MSBUILD% /v:m /nologo uploadrelease.proj /t:ShowHashes /p:Platform=x64 /p:IncludeDoc=false -if errorlevel 1 goto :failed - -popd -exit /B 0 - -:test -%MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=%1 -if errorlevel 1 ( - echo Test failed - purging and retrying - %MSBUILD% /v:m /nologo uploadrelease.proj /t:Purge - if errorlevel 1 exit /B - %MSBUILD% /v:m /nologo uploadrelease.proj /t:Test /p:Platform=%1 -) -exit /B - -:failed -popd -exit /B - -:Help -echo uploadrelease.bat --host HOST --user USERNAME [--target TARGET] [--dry-run] [-h] -echo. -echo --host (-o) Specify the upload host (required) -echo --user (-u) Specify the user on the host (required) -echo --target (-t) Specify the target directory on the host -echo --dry-run Display commands and filenames without executing them -echo --skip-gpg Does not generate GPG signatures before uploading -echo --skip-purge Does not perform CDN purge after uploading -echo --skip-test (-T) Does not perform post-upload tests -echo -h Display this help information -echo. diff --git a/Tools/msi/uploadrelease.proj b/Tools/msi/uploadrelease.proj deleted file mode 100644 index e70ae1de22fa78b..000000000000000 --- a/Tools/msi/uploadrelease.proj +++ /dev/null @@ -1,113 +0,0 @@ - - - - {2D69F2AB-D5D0-4344-84B5-EF6DB34A9BC9} - python - - - $(TARGET) - /srv/www.python.org/ftp/python - true - true - false - - - - - - - $(DownloadUrlBase.TrimEnd(`/`))/$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber) - $(DownloadUrl.Replace(`{version}`, `$(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber)`).Replace(`{arch}`, `$(ArchName)`).Replace(`{releasename}`, `$(ReleaseLevelName)`).Replace(`{msi}`, ``).TrimEnd(`/`)) - - - - - $(MSITarget) - - - $(EXETarget) - - - $(EXETarget) - - - - - - - - - - - - - - - - - %(File.CopyTo) - - - - - - - - - - - - - - - - - - - - - $(TEMP)\%(Filename)_source - $(TEMP)\%(Filename)_source\%(Filename)%(Extension) - $(TEMP)\%(Filename)_layout - $(OutputPath)\%(Filename)_layoutlog - $(OutputPath)\%(Filename)_layoutlog\%(Filename).log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/msi/uploadrelease.ps1 b/Tools/msi/uploadrelease.ps1 deleted file mode 100644 index a8669e5d65ed14b..000000000000000 --- a/Tools/msi/uploadrelease.ps1 +++ /dev/null @@ -1,171 +0,0 @@ -<# -.Synopsis - Uploads from a VSTS release build layout to python.org -.Description - Given the downloaded/extracted build artifact from a release - build run on python.visualstudio.com, this script uploads - the files to the correct locations. -.Parameter build - The location on disk of the extracted build artifact. -.Parameter user - The username to use when logging into the host. -.Parameter server - The host or PuTTY session name. -.Parameter target - The subdirectory on the host to copy files to. -.Parameter tests - The path to run download tests in. -.Parameter doc_htmlhelp - Optional path besides -build to locate CHM files. -.Parameter embed - Optional path besides -build to locate ZIP files. -.Parameter skipupload - Skip uploading -.Parameter skippurge - Skip purging the CDN -.Parameter skiptest - Skip the download tests -.Parameter skiphash - Skip displaying hashes -#> -param( - [Parameter(Mandatory=$true)][string]$build, - [Parameter(Mandatory=$true)][string]$user, - [string]$server="python-downloads", - [string]$target="/srv/www.python.org/ftp/python", - [string]$tests=${env:TEMP}, - [string]$doc_htmlhelp=$null, - [string]$embed=$null, - [switch]$skipupload, - [switch]$skippurge, - [switch]$skiptest, - [switch]$skiphash -) - -if (-not $build) { throw "-build option is required" } -if (-not $user) { throw "-user option is required" } - -$tools = $script:MyInvocation.MyCommand.Path | Split-Path -parent; - -if (-not ((Test-Path "$build\win32\python-*.exe") -or (Test-Path "$build\amd64\python-*.exe"))) { - throw "-build argument does not look like a 'build' directory" -} - -function find-putty-tool { - param ([string]$n) - $t = gcm $n -EA 0 - if (-not $t) { $t = gcm ".\$n" -EA 0 } - if (-not $t) { $t = gcm "${env:ProgramFiles}\PuTTY\$n" -EA 0 } - if (-not $t) { $t = gcm "${env:ProgramFiles(x86)}\PuTTY\$n" -EA 0 } - if (-not $t) { throw "Unable to locate $n.exe. Please put it on $PATH" } - return gi $t.Path -} - -$p = gci -r "$build\python-*.exe" | ` - ?{ $_.Name -match '^python-(\d+\.\d+\.\d+)((a|b|rc)\d+)?-.+' } | ` - select -first 1 | ` - %{ $Matches[1], $Matches[2] } - -"Uploading version $($p[0]) $($p[1])" -" from: $build" -" to: $($server):$target/$($p[0])" -"" - -if (-not $skipupload) { - # Upload files to the server - $pscp = find-putty-tool "pscp" - $plink = find-putty-tool "plink" - - "Upload using $pscp and $plink" - "" - - if ($doc_htmlhelp) { - $chm = gci -EA 0 $doc_htmlhelp\python*.chm, $doc_htmlhelp\python*.chm.asc - } else { - $chm = gci -EA 0 $build\python*.chm, $build\python*.chm.asc - } - - $d = "$target/$($p[0])/" - & $plink -batch $user@$server mkdir $d - & $plink -batch $user@$server chgrp downloads $d - & $plink -batch $user@$server chmod o+rx $d - if ($chm) { - & $pscp -batch $chm.FullName "$user@${server}:$d" - if (-not $?) { throw "Failed to upload $chm" } - } - - $dirs = gci "$build" -Directory - if ($embed) { - $dirs = ($dirs, (gi $embed)) | %{ $_ } - } - - foreach ($a in $dirs) { - "Uploading files from $($a.FullName)" - pushd "$($a.FullName)" - $exe = gci *.exe, *.exe.asc, *.zip, *.zip.asc - $msi = gci *.msi, *.msi.asc, *.msu, *.msu.asc - popd - - if ($exe) { - & $pscp -batch $exe.FullName "$user@${server}:$d" - if (-not $?) { throw "Failed to upload $exe" } - } - - if ($msi) { - $sd = "$d$($a.Name)$($p[1])/" - & $plink -batch $user@$server mkdir $sd - & $plink -batch $user@$server chgrp downloads $sd - & $plink -batch $user@$server chmod o+rx $sd - & $pscp -batch $msi.FullName "$user@${server}:$sd" - if (-not $?) { throw "Failed to upload $msi" } - & $plink -batch $user@$server chgrp downloads $sd* - & $plink -batch $user@$server chmod g-x,o+r $sd* - } - } - - & $plink -batch $user@$server chgrp downloads $d* - & $plink -batch $user@$server chmod g-x,o+r $d* - & $pscp -ls "$user@${server}:$d" -} - -if (-not $skippurge) { - # Run a CDN purge - py $tools\purge.py "$($p[0])$($p[1])" -} - -if (-not $skiptest) { - # Use each web installer to produce a layout. This will download - # each referenced file and validate their signatures/hashes. - gci "$build\*-webinstall.exe" -r -File | %{ - $d = mkdir "$tests\$($_.BaseName)" -Force - gci $d -r -File | del - $ic = copy $_ $d -PassThru - "Checking layout for $($ic.Name)" - Start-Process -wait $ic "/passive", "/layout", "$d\layout", "/log", "$d\log\install.log" - if (-not $?) { - Write-Error "Failed to validate layout of $($inst.Name)" - } - } -} - -if (-not $skiphash) { - # Display MD5 hash and size of each downloadable file - pushd $build - $files = gci python*.chm, *\*.exe, *\*.zip - if ($doc_htmlhelp) { - cd $doc_htmlhelp - $files = ($files, (gci python*.chm)) | %{ $_ } - } - if ($embed) { - cd $embed - $files = ($files, (gci *.zip)) | %{ $_ } - } - popd - - $hashes = $files | ` - Sort-Object Name | ` - Format-Table Name, @{Label="MD5"; Expression={(Get-FileHash $_ -Algorithm MD5).Hash}}, Length -AutoSize | ` - Out-String -Width 4096 - $hashes | clip - $hashes -} diff --git a/Tools/msi/wix.props b/Tools/msi/wix.props deleted file mode 100644 index 707c8aeacb648dc..000000000000000 --- a/Tools/msi/wix.props +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - $(MSBuildThisFileDirectory)\Wix\ - $(ExternalsDir)\windows-installer\wix-314\ - $(WixInstallPath)\Wix.targets - - \ No newline at end of file diff --git a/Tools/nuget/build.bat b/Tools/nuget/build.bat deleted file mode 100644 index b532bd74216831e..000000000000000 --- a/Tools/nuget/build.bat +++ /dev/null @@ -1,70 +0,0 @@ -@echo off -setlocal -set D=%~dp0 -set PCBUILD=%D%..\..\PCbuild\ -if "%Py_OutDir%"=="" set Py_OutDir=%PCBUILD% - -set BUILDX86= -set BUILDX64= -set BUILDARM32= -set REBUILD= -set OUTPUT= -set PACKAGES= -set PYTHON_EXE= - -:CheckOpts -if "%~1" EQU "-h" goto Help -if "%~1" EQU "-x86" (set BUILDX86=1) && shift && goto CheckOpts -if "%~1" EQU "-x64" (set BUILDX64=1) && shift && goto CheckOpts -if "%~1" EQU "-arm32" (set BUILDARM32=1) && shift && goto CheckOpts -if "%~1" EQU "-r" (set REBUILD=-r) && shift && goto CheckOpts -if "%~1" EQU "-o" (set OUTPUT="/p:OutputPath=%~2") && shift && shift && goto CheckOpts -if "%~1" EQU "--out" (set OUTPUT="/p:OutputPath=%~2") && shift && shift && goto CheckOpts -if "%~1" EQU "-p" (set PACKAGES=%PACKAGES% %~2) && shift && shift && goto CheckOpts -if "%~1" EQU "--python-exe" (set PYTHON_EXE="/p:PythonExe=%~2") && shift && shift && goto CheckOpts - -if not defined BUILDX86 if not defined BUILDX64 if not defined BUILDARM32 (set BUILDX86=1) && (set BUILDX64=1) && (set BUILDARM32=1) - -call "%D%..\msi\get_externals.bat" -call "%PCBUILD%find_msbuild.bat" %MSBUILD% -if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable & exit /b 2) - -if defined PACKAGES set PACKAGES="/p:Packages=%PACKAGES%" - -if defined BUILDX86 ( - if defined REBUILD ( call "%PCBUILD%build.bat" -e -r - ) else if not exist "%Py_OutDir%win32\python.exe" call "%PCBUILD%build.bat" -e - if errorlevel 1 goto :eof - - %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x86 %OUTPUT% %PACKAGES% %PYTHON_EXE% - if errorlevel 1 goto :eof -) - -if defined BUILDX64 ( - if defined REBUILD ( call "%PCBUILD%build.bat" -p x64 -e -r - ) else if not exist "%Py_OutDir%amd64\python.exe" call "%PCBUILD%build.bat" -p x64 -e - if errorlevel 1 goto :eof - - %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=x64 %OUTPUT% %PACKAGES% %PYTHON_EXE% - if errorlevel 1 goto :eof -) - -if defined BUILDARM32 ( - if defined REBUILD ( call "%PCBUILD%build.bat" -p ARM -e -r --no-tkinter - ) else if not exist "%Py_OutDir%arm32\python.exe" call "%PCBUILD%build.bat" -p ARM -e --no-tkinter - if errorlevel 1 goto :eof - - %MSBUILD% "%D%make_pkg.proj" /p:Configuration=Release /p:Platform=ARM %OUTPUT% %PACKAGES% %PYTHON_EXE% - if errorlevel 1 goto :eof -) - -exit /B 0 - -:Help -echo build.bat [-x86] [-x64] [--out DIR] [-r] [-h] -echo. -echo -x86 Build x86 installers -echo -x64 Build x64 installers -echo -r Rebuild rather than incremental build -echo --out [DIR] Override output directory -echo -h Show usage diff --git a/Tools/nuget/make_pkg.proj b/Tools/nuget/make_pkg.proj deleted file mode 100644 index 710ef3dcb5c01de..000000000000000 --- a/Tools/nuget/make_pkg.proj +++ /dev/null @@ -1,63 +0,0 @@ - - - - {10487945-15D1-4092-A214-338395C4116B} - python - $(OutputName)x86 - $(OutputName)arm32 - $(OutputName)daily - - false - true - - - - - - $(ExternalsDir)\windows-installer\nuget\nuget.exe - $(MajorVersionNumber).$(MinorVersionNumber).$(MicroVersionNumber) - $(NuspecVersion)-$(ReleaseLevelName) - $(MajorVersionNumber).$(MinorVersionNumber).$(DailyBuildVersion) - false - $(OutputName).$(NuspecVersion) - .nupkg - $(IntermediateOutputPath)\nuget_$(ArchName)\ - - rmdir /q/s "$(IntermediateOutputPath.TrimEnd(`\`))" - - "$(PythonExe)" "$(PySourcePath)PC\layout" - $(PythonArguments) -b "$(BuildPath.TrimEnd(`\`))" -s "$(PySourcePath.TrimEnd(`\`))" - $(PythonArguments) -t "$(IntermediateOutputPath)obj" - $(PythonArguments) --copy "$(IntermediateOutputPath)pkg" - $(PythonArguments) --preset-nuget - - "$(IntermediateOutputPath)pkg\pip.exe" -B -m pip install -U $(Packages) - - "$(Nuget)" pack "$(IntermediateOutputPath)pkg\python.nuspec" -BasePath "$(IntermediateOutputPath)pkg" - "$(Nuget)" pack "$(MSBuildThisFileDirectory)\$(OutputName).symbols.nuspec" -BasePath "$(BuildPath.TrimEnd(`\`))" - $(NugetArguments) -OutputDirectory "$(OutputPath.Trim(`\`))" - $(NugetArguments) -Version "$(NuspecVersion)" - $(NugetArguments) -NoPackageAnalysis -NonInteractive - - $(Environment)%0D%0Aset PYTHONPATH=$(PySourcePath)Lib - $(Environment)%0D%0Aset PYTHON_NUSPEC_VERSION=$(NuspecVersion) - $(Environment)%0D%0Aset PYTHON_PROPS_PLATFORM=$(Platform) - $(Environment)%0D%0Aset PYTHON_PROPS_PLATFORM=Win32 - $(Environment)%0D%0Amkdir "$(OutputPath.Trim(`\`))" >nul 2>nul - - - - - - - - - - - - - - - - - diff --git a/Tools/nuget/python.nuspec b/Tools/nuget/python.nuspec deleted file mode 100644 index 2da5f2037eb2342..000000000000000 --- a/Tools/nuget/python.nuspec +++ /dev/null @@ -1,18 +0,0 @@ - - - - python - Python - 0.0.0.0 - Python Software Foundation - tools\LICENSE.txt - https://www.python.org/ - Installs 64-bit Python for use in build scenarios. - https://www.python.org/static/favicon.ico - python - - - - - - diff --git a/Tools/nuget/pythonarm32.nuspec b/Tools/nuget/pythonarm32.nuspec deleted file mode 100644 index 2d197931edb32c3..000000000000000 --- a/Tools/nuget/pythonarm32.nuspec +++ /dev/null @@ -1,20 +0,0 @@ - - - - pythonarm32 - Python (ARM32) - Python Software Foundation - 0.0.0.0 - https://docs.python.org/3/license.html - https://www.python.org/ - false - Installs Python ARM32 for use in build scenarios. - https://www.python.org/static/favicon.ico - python - - - - - - - diff --git a/Tools/nuget/pythondaily.nuspec b/Tools/nuget/pythondaily.nuspec deleted file mode 100644 index 7df1983f42a6fe2..000000000000000 --- a/Tools/nuget/pythondaily.nuspec +++ /dev/null @@ -1,19 +0,0 @@ - - - - pythondaily - Python (Daily build) - Python Software Foundation - 0.0.0.0 - tools\LICENSE.txt - https://www.python.org/ - Installs an unsigned, untested build of Python for test purposes only. - https://www.python.org/static/favicon.ico - python - - - - - - - diff --git a/Tools/nuget/pythondaily.symbols.nuspec b/Tools/nuget/pythondaily.symbols.nuspec deleted file mode 100644 index 608e15c50e02d2d..000000000000000 --- a/Tools/nuget/pythondaily.symbols.nuspec +++ /dev/null @@ -1,22 +0,0 @@ - - - - pythondaily.symbols - Python (Daily build) - 0.0.0.0 - Python Software Foundation - https://www.python.org/ - Contains symbols for the daily build of Python. - https://www.python.org/static/favicon.ico - python - - - - - - - - - - - diff --git a/Tools/nuget/pythonx86.nuspec b/Tools/nuget/pythonx86.nuspec deleted file mode 100644 index ea878ba0bf390b6..000000000000000 --- a/Tools/nuget/pythonx86.nuspec +++ /dev/null @@ -1,19 +0,0 @@ - - - - pythonx86 - Python (32-bit) - Python Software Foundation - 0.0.0.0 - tools\LICENSE.txt - https://www.python.org/ - Installs 32-bit Python for use in build scenarios. - https://www.python.org/static/favicon.ico - python - - - - - - - diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 46381ea58a12382..900c5028b0af604 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==2.1.0 +mypy==2.3.0 # needed for peg_generator: -types-psutil==7.2.2.20260508 -types-setuptools==82.0.0.20260508 +types-psutil==7.2.2.20260518 +types-setuptools==83.0.0.20260724 diff --git a/configure b/configure index c8494891087d045..69d76f7b9f2dba2 100755 --- a/configure +++ b/configure @@ -1102,6 +1102,7 @@ enable_framework with_app_store_compliance enable_wasm_dynamic_linking enable_wasm_pthreads +enable_emscripten_syscalls with_suffix enable_shared with_static_libpython @@ -1147,6 +1148,7 @@ enable_big_digits with_platlibdir with_wheel_pkg_dir with_readline +with_curses with_computed_gotos with_tail_call_interp with_remote_debug @@ -1842,6 +1844,10 @@ Optional Features: loader to handle imports --enable-wasm-pthreads Enable pthread emulation for WebAssembly (default is no) + --disable-emscripten-syscalls + Disable the Emscripten syscall overrides in + Python/emscripten_syscalls.c (default is enabled for + Emscripten) --enable-shared enable building a shared Python library (default is no) --enable-static-libpython-for-interpreter @@ -1969,6 +1975,10 @@ Optional Packages: (default: none) --with(out)-readline[=editline|readline|no] use libedit for backend or disable readline module + --with(out)-curses[=ncursesw|ncurses|curses|no] + select the curses backend for the curses and + _curses_panel modules, or disable them (default: + auto) --with-computed-gotos enable computed gotos in evaluation loop (enabled by default on supported compilers) --with-tail-call-interp enable tail-calling interpreter in evaluation loop @@ -4804,6 +4814,13 @@ printf "%s\n" "#define _XOPEN_SOURCE_EXTENDED 1" >>confdefs.h printf "%s\n" "#define _POSIX_C_SOURCE 202405L" >>confdefs.h + + # Defining _POSIX_C_SOURCE and _XOPEN_SOURCE hides C23 library + # declarations on FreeBSD (e.g. sinpi() in math.h) when compiling + # with -std=c11. Defining _ISOC23_SOURCE makes them visible again. + +printf "%s\n" "#define _ISOC23_SOURCE 1" >>confdefs.h + fi # On HP-UX mbstate_t requires _INCLUDE__STDC_A1_SOURCE @@ -6390,6 +6407,8 @@ else case e in #( cat > conftest.c <&5 @@ -6969,7 +6988,117 @@ else CXX="$ac_cv_path_CXX" fi ;; - icc) if test -n "$ac_tool_prefix"; then + icx) if test -n "$ac_tool_prefix"; then + # Extract the first word of "${ac_tool_prefix}icpx", so it can be a program name with args. +set dummy ${ac_tool_prefix}icpx; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_path_CXX+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) case $CXX in + [\\/]* | ?:[\\/]*) + ac_cv_path_CXX="$CXX" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_path_CXX="$as_dir$ac_word$ac_exec_ext" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac ;; +esac +fi +CXX=$ac_cv_path_CXX +if test -n "$CXX"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $CXX" >&5 +printf "%s\n" "$CXX" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + +fi +if test -z "$ac_cv_path_CXX"; then + ac_pt_CXX=$CXX + # Extract the first word of "icpx", so it can be a program name with args. +set dummy icpx; ac_word=$2 +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 +printf %s "checking for $ac_word... " >&6; } +if test ${ac_cv_path_ac_pt_CXX+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) case $ac_pt_CXX in + [\\/]* | ?:[\\/]*) + ac_cv_path_ac_pt_CXX="$ac_pt_CXX" # Let the user override the test with a path. + ;; + *) + as_save_IFS=$IFS; IFS=$PATH_SEPARATOR +for as_dir in $PATH +do + IFS=$as_save_IFS + case $as_dir in #((( + '') as_dir=./ ;; + */) ;; + *) as_dir=$as_dir/ ;; + esac + for ac_exec_ext in '' $ac_executable_extensions; do + if as_fn_executable_p "$as_dir$ac_word$ac_exec_ext"; then + ac_cv_path_ac_pt_CXX="$as_dir$ac_word$ac_exec_ext" + printf "%s\n" "$as_me:${as_lineno-$LINENO}: found $as_dir$ac_word$ac_exec_ext" >&5 + break 2 + fi +done + done +IFS=$as_save_IFS + + ;; +esac ;; +esac +fi +ac_pt_CXX=$ac_cv_path_ac_pt_CXX +if test -n "$ac_pt_CXX"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_pt_CXX" >&5 +printf "%s\n" "$ac_pt_CXX" >&6; } +else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi + + if test "x$ac_pt_CXX" = x; then + CXX="notfound" + else + case $cross_compiling:$ac_tool_warned in +yes:) +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: using cross tools not prefixed with host triplet" >&5 +printf "%s\n" "$as_me: WARNING: using cross tools not prefixed with host triplet" >&2;} +ac_tool_warned=yes ;; +esac + CXX=$ac_pt_CXX + fi +else + CXX="$ac_cv_path_CXX" +fi + ;; + icc) if test -n "$ac_tool_prefix"; then # Extract the first word of "${ac_tool_prefix}icpc", so it can be a program name with args. set dummy ${ac_tool_prefix}icpc; ac_word=$2 { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 @@ -7266,6 +7395,12 @@ case $ac_sys_system in #( ;; esac +if test x$SOABI_PLATFORM != x; then + +printf "%s\n" "#define SOABI_PLATFORM \"${SOABI_PLATFORM}\"" >>confdefs.h + +fi + if test x$MULTIARCH != x; then MULTIARCH_CPPFLAGS="-DMULTIARCH=\\\"$MULTIARCH\\\"" fi @@ -7496,6 +7631,30 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_wasm_pthreads" >&5 printf "%s\n" "$enable_wasm_pthreads" >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --enable-emscripten-syscalls" >&5 +printf %s "checking for --enable-emscripten-syscalls... " >&6; } +# Check whether --enable-emscripten-syscalls was given. +if test ${enable_emscripten_syscalls+y} +then : + enableval=$enable_emscripten_syscalls; + case $ac_sys_system in #( + Emscripten) : + ;; #( + *) : + as_fn_error $? "--enable-emscripten-syscalls only applies to Emscripten" "$LINENO" 5 + ;; +esac + +else case e in #( + e) + enable_emscripten_syscalls=yes + ;; +esac +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_emscripten_syscalls" >&5 +printf "%s\n" "$enable_emscripten_syscalls" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-suffix" >&5 printf %s "checking for --with-suffix... " >&6; } @@ -9117,7 +9276,7 @@ fi LLVM_PROF_ERR=no case "$ac_cv_cc_name" in - clang) + clang|icx) # Any changes made here should be reflected in the GCC+Darwin case below PGO_PROF_GEN_FLAG="-fprofile-instr-generate" PGO_PROF_USE_FLAG="-fprofile-instr-use=\"\$(shell pwd)/code.profclangd\"" @@ -9802,7 +9961,6 @@ fi as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" as_fn_append LINKFORSHARED " -sTEXTDECODER=2" - as_fn_append LINKFORSHARED " -sGROWABLE_ARRAYBUFFERS=0" if test "x$enable_wasm_dynamic_linking" = xyes then : @@ -11404,6 +11562,10 @@ case "$ac_cv_cc_name" in mpicc) CFLAGS_NODIST="$CFLAGS_NODIST" ;; +icx) + # ICX needs fp-model=precise (the default in clang) or floats behave badly + CFLAGS_NODIST="$CFLAGS_NODIST -ffp-model=precise" + ;; icc) # ICC needs -fp-model strict or floats behave badly CFLAGS_NODIST="$CFLAGS_NODIST -fp-model strict" @@ -15283,9 +15445,10 @@ then : fi -# gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. +# gh-124228: While the libuuid library is available on NetBSD and OpenBSD, +# it supports only UUID version 4. # This restriction inhibits the proper generation of time-based UUIDs. -if test "$ac_sys_system" = "NetBSD"; then +if test "$ac_sys_system" = "NetBSD" || test "$ac_sys_system" = "OpenBSD"; then have_uuid=missing printf "%s\n" "#define HAVE_UUID_H 0" >>confdefs.h @@ -20031,7 +20194,13 @@ PLATFORM_OBJS= case $ac_sys_system in #( Emscripten) : - as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_trampoline_wasm.o Python/emscripten_syscalls.o' + as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_trampoline_wasm.o' + if test "x$enable_emscripten_syscalls" = xyes +then : + + as_fn_append PLATFORM_OBJS ' Python/emscripten_syscalls.o' + +fi as_fn_append PLATFORM_HEADERS ' $(srcdir)/Include/internal/pycore_emscripten_signal.h $(srcdir)/Include/internal/pycore_emscripten_trampoline.h' ;; #( *) : @@ -26938,8 +27107,8 @@ esac # rounding precision of 64 bits. For gcc/x86, we can fix this by # using inline assembler to get and set the x87 FPU control word. -# This inline assembler syntax may also work for suncc and icc, -# so we try it on all platforms. +# This inline assembler syntax works for icx and may also work for +# suncc and icc, so we try it on all platforms. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether we can use gcc inline assembler to get and set x87 control word" >&5 printf %s "checking whether we can use gcc inline assembler to get and set x87 control word... " >&6; } @@ -29281,9 +29450,36 @@ fi have_curses=no have_panel=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for --with-curses" >&5 +printf %s "checking for --with-curses... " >&6; } + +# Check whether --with-curses was given. +if test ${with_curses+y} +then : + withval=$with_curses; case $with_curses in #( + yes|auto) : + with_curses=auto ;; #( + ncursesw|ncurses|curses|no) : + ;; #( + *) : + as_fn_error $? "proper usage is --with(out)-curses[=ncursesw|ncurses|curses|no]" "$LINENO" 5 ;; +esac +else case e in #( + e) with_curses=auto ;; +esac +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_curses" >&5 +printf "%s\n" "$with_curses" >&6; } + -# Check for ncursesw/panelw first. If that fails, try ncurses/panel. +# Detect the selected backend. ncursesw/ncurses are found via pkg-config; +# native curses has no .pc file and is left to the header/link probes below. +# curses_libs/panel_libs drive the AC_SEARCH_LIBS fallback; for "no" they are +# empty so nothing links and have_curses stays "no". +case $with_curses in #( + ncursesw) : @@ -29418,43 +29614,21 @@ fi # Put the nasty error message in config.log where it belongs echo "$PANEL_PKG_ERRORS" >&5 - have_panel=no -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } - have_panel=no -else - PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS - PANEL_LIBS=$pkg_cv_PANEL_LIBS - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -printf "%s\n" "yes" >&6; } - -printf "%s\n" "#define HAVE_PANELW 1" >>confdefs.h - - have_panel=yes -fi -fi - - -if test "x$have_curses" = xno -then : - - pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncurses" >&5 -printf %s "checking for ncurses... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gnupanel" >&5 +printf %s "checking for gnupanel... " >&6; } -if test -n "$CURSES_CFLAGS"; then - pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" +if test -n "$PANEL_CFLAGS"; then + pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncurses" 2>/dev/null` + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "gnupanel" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -29462,16 +29636,16 @@ fi else pkg_failed=untried fi -if test -n "$CURSES_LIBS"; then - pkg_cv_CURSES_LIBS="$CURSES_LIBS" +if test -n "$PANEL_LIBS"; then + pkg_cv_PANEL_LIBS="$PANEL_LIBS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 - ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncurses" 2>/dev/null` + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "gnupanel" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -29492,42 +29666,43 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncurses" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gnupanel" 2>&1` else - CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncurses" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gnupanel" 2>&1` fi # Put the nasty error message in config.log where it belongs - echo "$CURSES_PKG_ERRORS" >&5 + echo "$PANEL_PKG_ERRORS" >&5 - have_curses=no + have_panel=no elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - have_curses=no + have_panel=no else - CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS - CURSES_LIBS=$pkg_cv_CURSES_LIBS + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } - -printf "%s\n" "#define HAVE_NCURSES 1" >>confdefs.h - - have_curses=yes + have_panel=yes +fi +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panel" >&5 -printf %s "checking for panel... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gnupanel" >&5 +printf %s "checking for gnupanel... " >&6; } if test -n "$PANEL_CFLAGS"; then pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panel\""; } >&5 - ($PKG_CONFIG --exists --print-errors "panel") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "panel" 2>/dev/null` + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "gnupanel" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -29539,12 +29714,12 @@ if test -n "$PANEL_LIBS"; then pkg_cv_PANEL_LIBS="$PANEL_LIBS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panel\""; } >&5 - ($PKG_CONFIG --exists --print-errors "panel") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "panel" 2>/dev/null` + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "gnupanel" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -29565,9 +29740,9 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panel" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gnupanel" 2>&1` else - PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panel" 2>&1` + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gnupanel" 2>&1` fi # Put the nasty error message in config.log where it belongs echo "$PANEL_PKG_ERRORS" >&5 @@ -29577,256 +29752,1747 @@ elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_panel=yes +fi else PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS PANEL_LIBS=$pkg_cv_PANEL_LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } -printf "%s\n" "#define HAVE_PANEL 1" >>confdefs.h +printf "%s\n" "#define HAVE_PANELW 1" >>confdefs.h have_panel=yes fi fi -fi - -save_CFLAGS=$CFLAGS -save_CPPFLAGS=$CPPFLAGS -save_LDFLAGS=$LDFLAGS -save_LIBS=$LIBS + curses_libs="ncursesw"; panel_libs="panelw gnupanel" ;; #( + ncurses) : - # Make sure we've got the header defines. - as_fn_append CPPFLAGS " $CURSES_CFLAGS $PANEL_CFLAGS" - ac_fn_c_check_header_compile "$LINENO" "ncursesw/curses.h" "ac_cv_header_ncursesw_curses_h" "$ac_includes_default" -if test "x$ac_cv_header_ncursesw_curses_h" = xyes -then : - printf "%s\n" "#define HAVE_NCURSESW_CURSES_H 1" >>confdefs.h -fi -ac_fn_c_check_header_compile "$LINENO" "ncursesw/ncurses.h" "ac_cv_header_ncursesw_ncurses_h" "$ac_includes_default" -if test "x$ac_cv_header_ncursesw_ncurses_h" = xyes -then : - printf "%s\n" "#define HAVE_NCURSESW_NCURSES_H 1" >>confdefs.h +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncurses" >&5 +printf %s "checking for ncurses... " >&6; } +if test -n "$CURSES_CFLAGS"; then + pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncurses" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes fi -ac_fn_c_check_header_compile "$LINENO" "ncursesw/panel.h" "ac_cv_header_ncursesw_panel_h" "$ac_includes_default" -if test "x$ac_cv_header_ncursesw_panel_h" = xyes -then : - printf "%s\n" "#define HAVE_NCURSESW_PANEL_H 1" >>confdefs.h - + else + pkg_failed=untried fi -ac_fn_c_check_header_compile "$LINENO" "ncurses/curses.h" "ac_cv_header_ncurses_curses_h" "$ac_includes_default" +if test -n "$CURSES_LIBS"; then + pkg_cv_CURSES_LIBS="$CURSES_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncurses" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncurses" 2>&1` + else + CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncurses" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$CURSES_PKG_ERRORS" >&5 + + have_curses=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_curses=no +else + CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS + CURSES_LIBS=$pkg_cv_CURSES_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +printf "%s\n" "#define HAVE_NCURSES 1" >>confdefs.h + + have_curses=yes + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panel" >&5 +printf %s "checking for panel... " >&6; } + +if test -n "$PANEL_CFLAGS"; then + pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "panel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "panel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$PANEL_LIBS"; then + pkg_cv_PANEL_LIBS="$PANEL_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "panel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "panel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panel" 2>&1` + else + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panel" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 + + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gnupanel" >&5 +printf %s "checking for gnupanel... " >&6; } + +if test -n "$PANEL_CFLAGS"; then + pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$PANEL_LIBS"; then + pkg_cv_PANEL_LIBS="$PANEL_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gnupanel" 2>&1` + else + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gnupanel" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 + + have_panel=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_panel=yes +fi +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gnupanel" >&5 +printf %s "checking for gnupanel... " >&6; } + +if test -n "$PANEL_CFLAGS"; then + pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$PANEL_LIBS"; then + pkg_cv_PANEL_LIBS="$PANEL_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gnupanel" 2>&1` + else + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gnupanel" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 + + have_panel=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_panel=yes +fi +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +printf "%s\n" "#define HAVE_PANEL 1" >>confdefs.h + + have_panel=yes +fi +fi + + + curses_libs="ncurses"; panel_libs="panel gnupanel" ;; #( + curses) : + curses_libs="curses"; panel_libs="panel" ;; #( + no) : + curses_libs=""; panel_libs="" ;; #( + *) : + + + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncursesw" >&5 +printf %s "checking for ncursesw... " >&6; } + +if test -n "$CURSES_CFLAGS"; then + pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncursesw\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncursesw") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncursesw" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$CURSES_LIBS"; then + pkg_cv_CURSES_LIBS="$CURSES_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncursesw\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncursesw") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncursesw" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncursesw" 2>&1` + else + CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncursesw" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$CURSES_PKG_ERRORS" >&5 + + have_curses=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_curses=no +else + CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS + CURSES_LIBS=$pkg_cv_CURSES_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h + + have_curses=yes + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panelw" >&5 +printf %s "checking for panelw... " >&6; } + +if test -n "$PANEL_CFLAGS"; then + pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panelw\""; } >&5 + ($PKG_CONFIG --exists --print-errors "panelw") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "panelw" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$PANEL_LIBS"; then + pkg_cv_PANEL_LIBS="$PANEL_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panelw\""; } >&5 + ($PKG_CONFIG --exists --print-errors "panelw") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "panelw" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panelw" 2>&1` + else + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panelw" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 + + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gnupanel" >&5 +printf %s "checking for gnupanel... " >&6; } + +if test -n "$PANEL_CFLAGS"; then + pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$PANEL_LIBS"; then + pkg_cv_PANEL_LIBS="$PANEL_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gnupanel" 2>&1` + else + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gnupanel" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 + + have_panel=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_panel=yes +fi +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gnupanel" >&5 +printf %s "checking for gnupanel... " >&6; } + +if test -n "$PANEL_CFLAGS"; then + pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$PANEL_LIBS"; then + pkg_cv_PANEL_LIBS="$PANEL_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gnupanel" 2>&1` + else + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gnupanel" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 + + have_panel=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_panel=yes +fi +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +printf "%s\n" "#define HAVE_PANELW 1" >>confdefs.h + + have_panel=yes +fi +fi + + + if test "x$have_curses" = xno +then : + + + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncurses" >&5 +printf %s "checking for ncurses... " >&6; } + +if test -n "$CURSES_CFLAGS"; then + pkg_cv_CURSES_CFLAGS="$CURSES_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CURSES_CFLAGS=`$PKG_CONFIG --cflags "ncurses" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$CURSES_LIBS"; then + pkg_cv_CURSES_LIBS="$CURSES_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"ncurses\""; } >&5 + ($PKG_CONFIG --exists --print-errors "ncurses") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_CURSES_LIBS=`$PKG_CONFIG --libs "ncurses" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + CURSES_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "ncurses" 2>&1` + else + CURSES_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "ncurses" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$CURSES_PKG_ERRORS" >&5 + + have_curses=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_curses=no +else + CURSES_CFLAGS=$pkg_cv_CURSES_CFLAGS + CURSES_LIBS=$pkg_cv_CURSES_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +printf "%s\n" "#define HAVE_NCURSES 1" >>confdefs.h + + have_curses=yes + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for panel" >&5 +printf %s "checking for panel... " >&6; } + +if test -n "$PANEL_CFLAGS"; then + pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "panel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "panel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$PANEL_LIBS"; then + pkg_cv_PANEL_LIBS="$PANEL_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"panel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "panel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "panel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "panel" 2>&1` + else + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "panel" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 + + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gnupanel" >&5 +printf %s "checking for gnupanel... " >&6; } + +if test -n "$PANEL_CFLAGS"; then + pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$PANEL_LIBS"; then + pkg_cv_PANEL_LIBS="$PANEL_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gnupanel" 2>&1` + else + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gnupanel" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 + + have_panel=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_panel=yes +fi +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +pkg_failed=no +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for gnupanel" >&5 +printf %s "checking for gnupanel... " >&6; } + +if test -n "$PANEL_CFLAGS"; then + pkg_cv_PANEL_CFLAGS="$PANEL_CFLAGS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_CFLAGS=`$PKG_CONFIG --cflags "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi +if test -n "$PANEL_LIBS"; then + pkg_cv_PANEL_LIBS="$PANEL_LIBS" + elif test -n "$PKG_CONFIG"; then + if test -n "$PKG_CONFIG" && \ + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnupanel\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnupanel") 2>&5 + ac_status=$? + printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; }; then + pkg_cv_PANEL_LIBS=`$PKG_CONFIG --libs "gnupanel" 2>/dev/null` + test "x$?" != "x0" && pkg_failed=yes +else + pkg_failed=yes +fi + else + pkg_failed=untried +fi + + + +if test $pkg_failed = yes; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +if $PKG_CONFIG --atleast-pkgconfig-version 0.20; then + _pkg_short_errors_supported=yes +else + _pkg_short_errors_supported=no +fi + if test $_pkg_short_errors_supported = yes; then + PANEL_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "gnupanel" 2>&1` + else + PANEL_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "gnupanel" 2>&1` + fi + # Put the nasty error message in config.log where it belongs + echo "$PANEL_PKG_ERRORS" >&5 + + have_panel=no +elif test $pkg_failed = untried; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + have_panel=no +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + have_panel=yes +fi +else + PANEL_CFLAGS=$pkg_cv_PANEL_CFLAGS + PANEL_LIBS=$pkg_cv_PANEL_LIBS + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +printf "%s\n" "#define HAVE_PANEL 1" >>confdefs.h + + have_panel=yes +fi +fi + + +fi + curses_libs="ncursesw ncurses"; panel_libs="panelw panel gnupanel" ;; +esac + +save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + # Make sure we've got the header defines. For the native "curses" backend, + # probe only the plain headers: a system may also have ncurses headers (e.g. + # ncurses/curses.h), and picking those while linking the native library mixes + # incompatible declarations (e.g. tparm()) with the native . + as_fn_append CPPFLAGS " $CURSES_CFLAGS $PANEL_CFLAGS" + if test "x$with_curses" = xcurses +then : + ac_fn_c_check_header_compile "$LINENO" "curses.h" "ac_cv_header_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "panel.h" "ac_cv_header_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_PANEL_H 1" >>confdefs.h + +fi + +else case e in #( + e) ac_fn_c_check_header_compile "$LINENO" "ncursesw/curses.h" "ac_cv_header_ncursesw_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncursesw/ncurses.h" "ac_cv_header_ncursesw_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_NCURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncursesw/panel.h" "ac_cv_header_ncursesw_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_ncursesw_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW_PANEL_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/curses.h" "ac_cv_header_ncurses_curses_h" "$ac_includes_default" if test "x$ac_cv_header_ncurses_curses_h" = xyes then : - printf "%s\n" "#define HAVE_NCURSES_CURSES_H 1" >>confdefs.h + printf "%s\n" "#define HAVE_NCURSES_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/ncurses.h" "ac_cv_header_ncurses_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_NCURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses/panel.h" "ac_cv_header_ncurses_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_PANEL_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "curses.h" "ac_cv_header_curses_h" "$ac_includes_default" +if test "x$ac_cv_header_curses_h" = xyes +then : + printf "%s\n" "#define HAVE_CURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "ncurses.h" "ac_cv_header_ncurses_h" "$ac_includes_default" +if test "x$ac_cv_header_ncurses_h" = xyes +then : + printf "%s\n" "#define HAVE_NCURSES_H 1" >>confdefs.h + +fi +ac_fn_c_check_header_compile "$LINENO" "panel.h" "ac_cv_header_panel_h" "$ac_includes_default" +if test "x$ac_cv_header_panel_h" = xyes +then : + printf "%s\n" "#define HAVE_PANEL_H 1" >>confdefs.h + +fi + ;; +esac +fi + + # Check that we're able to link with crucial curses/panel functions. This + # also serves as a fallback in case pkg-config failed. + as_fn_append LIBS " $CURSES_LIBS $PANEL_LIBS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing initscr" >&5 +printf %s "checking for library containing initscr... " >&6; } +if test ${ac_cv_search_initscr+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char initscr (void); +int +main (void) +{ +return initscr (); + ; + return 0; +} +_ACEOF +for ac_lib in '' $curses_libs +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" +then : + ac_cv_search_initscr=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext + if test ${ac_cv_search_initscr+y} +then : + break +fi +done +if test ${ac_cv_search_initscr+y} +then : + +else case e in #( + e) ac_cv_search_initscr=no ;; +esac +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_initscr" >&5 +printf "%s\n" "$ac_cv_search_initscr" >&6; } +ac_res=$ac_cv_search_initscr +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + if test "x$have_curses" = xno +then : + have_curses=yes + CURSES_LIBS=${CURSES_LIBS-"$ac_cv_search_initscr"} +fi +else case e in #( + e) have_curses=no ;; +esac +fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing update_panels" >&5 +printf %s "checking for library containing update_panels... " >&6; } +if test ${ac_cv_search_update_panels+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char update_panels (void); +int +main (void) +{ +return update_panels (); + ; + return 0; +} +_ACEOF +for ac_lib in '' $panel_libs +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" +then : + ac_cv_search_update_panels=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext + if test ${ac_cv_search_update_panels+y} +then : + break +fi +done +if test ${ac_cv_search_update_panels+y} +then : + +else case e in #( + e) ac_cv_search_update_panels=no ;; +esac +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_update_panels" >&5 +printf "%s\n" "$ac_cv_search_update_panels" >&6; } +ac_res=$ac_cv_search_update_panels +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + if test "x$have_panel" = xno +then : + have_panel=yes + PANEL_LIBS=${PANEL_LIBS-"$ac_cv_search_update_panels"} +fi +else case e in #( + e) have_panel=no ;; +esac +fi + + + + +if test "$have_curses" != "no" +then : + +CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') + +if test "x$ac_sys_system" = xDarwin +then : + + + as_fn_append CURSES_CFLAGS " -D_XOPEN_SOURCE_EXTENDED=1" + +fi + +PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') + +case $with_curses in #( + ncursesw|no) : + ;; #( + *) : + + save_curses_cppflags=$CPPFLAGS + as_fn_append CPPFLAGS " $CURSES_CFLAGS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether curses supports wide characters" >&5 +printf %s "checking whether curses supports wide characters... " >&6; } +if test ${ac_cv_curses_wide+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + +int +main (void) +{ + + cchar_t wcval; + setcchar(&wcval, L"x", A_NORMAL, 0, NULL); + add_wch(&wcval); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_curses_wide=yes +else case e in #( + e) ac_cv_curses_wide=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_curses_wide" >&5 +printf "%s\n" "$ac_cv_curses_wide" >&6; } + CPPFLAGS=$save_curses_cppflags + if test "x$ac_cv_curses_wide" = xyes +then : + printf "%s\n" "#define HAVE_NCURSESW 1" >>confdefs.h + +fi + ;; +esac + +# On Solaris, term.h requires curses.h +ac_fn_c_check_header_compile "$LINENO" "term.h" "ac_cv_header_term_h" " +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + +" +if test "x$ac_cv_header_term_h" = xyes +then : + printf "%s\n" "#define HAVE_TERM_H 1" >>confdefs.h + +fi + + +# On HP/UX 11.0, mvwdelch is a block with a return statement +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether mvwdelch is an expression" >&5 +printf %s "checking whether mvwdelch is an expression... " >&6; } +if test ${ac_cv_mvwdelch_is_expression+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + +int +main (void) +{ + + int rtn; + rtn = mvwdelch(0,0,0); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_mvwdelch_is_expression=yes +else case e in #( + e) ac_cv_mvwdelch_is_expression=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_mvwdelch_is_expression" >&5 +printf "%s\n" "$ac_cv_mvwdelch_is_expression" >&6; } + +if test "$ac_cv_mvwdelch_is_expression" = yes +then + +printf "%s\n" "#define MVWDELCH_IS_EXPRESSION 1" >>confdefs.h + +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether WINDOW has _flags" >&5 +printf %s "checking whether WINDOW has _flags... " >&6; } +if test ${ac_cv_window_has_flags+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + +int +main (void) +{ + + WINDOW *w; + w->_flags = 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_window_has_flags=yes +else case e in #( + e) ac_cv_window_has_flags=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_window_has_flags" >&5 +printf "%s\n" "$ac_cv_window_has_flags" >&6; } + + +if test "$ac_cv_window_has_flags" = yes +then + +printf "%s\n" "#define WINDOW_HAS_FLAGS 1" >>confdefs.h + +fi + + + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_pad" >&5 +printf %s "checking for curses function is_pad... " >&6; } +if test ${ac_cv_lib_curses_is_pad+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + +int +main (void) +{ + + #ifndef is_pad + void *x=is_pad + #endif + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_is_pad=yes +else case e in #( + e) ac_cv_lib_curses_is_pad=no ;; +esac fi -ac_fn_c_check_header_compile "$LINENO" "ncurses/ncurses.h" "ac_cv_header_ncurses_ncurses_h" "$ac_includes_default" -if test "x$ac_cv_header_ncurses_ncurses_h" = xyes +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_pad" >&5 +printf "%s\n" "$ac_cv_lib_curses_is_pad" >&6; } + if test "x$ac_cv_lib_curses_is_pad" = xyes then : - printf "%s\n" "#define HAVE_NCURSES_NCURSES_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_CURSES_IS_PAD 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "ncurses/panel.h" "ac_cv_header_ncurses_panel_h" "$ac_includes_default" -if test "x$ac_cv_header_ncurses_panel_h" = xyes + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_term_resized" >&5 +printf %s "checking for curses function is_term_resized... " >&6; } +if test ${ac_cv_lib_curses_is_term_resized+y} then : - printf "%s\n" "#define HAVE_NCURSES_PANEL_H 1" >>confdefs.h + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + +int +main (void) +{ + + #ifndef is_term_resized + void *x=is_term_resized + #endif + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_is_term_resized=yes +else case e in #( + e) ac_cv_lib_curses_is_term_resized=no ;; +esac fi -ac_fn_c_check_header_compile "$LINENO" "curses.h" "ac_cv_header_curses_h" "$ac_includes_default" -if test "x$ac_cv_header_curses_h" = xyes +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_term_resized" >&5 +printf "%s\n" "$ac_cv_lib_curses_is_term_resized" >&6; } + if test "x$ac_cv_lib_curses_is_term_resized" = xyes then : - printf "%s\n" "#define HAVE_CURSES_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_CURSES_IS_TERM_RESIZED 1" >>confdefs.h fi -ac_fn_c_check_header_compile "$LINENO" "ncurses.h" "ac_cv_header_ncurses_h" "$ac_includes_default" -if test "x$ac_cv_header_ncurses_h" = xyes + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function resize_term" >&5 +printf %s "checking for curses function resize_term... " >&6; } +if test ${ac_cv_lib_curses_resize_term+y} then : - printf "%s\n" "#define HAVE_NCURSES_H 1" >>confdefs.h + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + +int +main (void) +{ + + #ifndef resize_term + void *x=resize_term + #endif + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_resize_term=yes +else case e in #( + e) ac_cv_lib_curses_resize_term=no ;; +esac fi -ac_fn_c_check_header_compile "$LINENO" "panel.h" "ac_cv_header_panel_h" "$ac_includes_default" -if test "x$ac_cv_header_panel_h" = xyes +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_resize_term" >&5 +printf "%s\n" "$ac_cv_lib_curses_resize_term" >&6; } + if test "x$ac_cv_lib_curses_resize_term" = xyes then : - printf "%s\n" "#define HAVE_PANEL_H 1" >>confdefs.h + +printf "%s\n" "#define HAVE_CURSES_RESIZE_TERM 1" >>confdefs.h fi - # Check that we're able to link with crucial curses/panel functions. This - # also serves as a fallback in case pkg-config failed. - as_fn_append LIBS " $CURSES_LIBS $PANEL_LIBS" - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing initscr" >&5 -printf %s "checking for library containing initscr... " >&6; } -if test ${ac_cv_search_initscr+y} + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function resizeterm" >&5 +printf %s "checking for curses function resizeterm... " >&6; } +if test ${ac_cv_lib_curses_resizeterm+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + +int +main (void) +{ + + #ifndef resizeterm + void *x=resizeterm + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_resizeterm=yes +else case e in #( + e) ac_cv_lib_curses_resizeterm=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_resizeterm" >&5 +printf "%s\n" "$ac_cv_lib_curses_resizeterm" >&6; } + if test "x$ac_cv_lib_curses_resizeterm" = xyes +then : + +printf "%s\n" "#define HAVE_CURSES_RESIZETERM 1" >>confdefs.h + +fi + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function immedok" >&5 +printf %s "checking for curses function immedok... " >&6; } +if test ${ac_cv_lib_curses_immedok+y} then : printf %s "(cached) " >&6 else case e in #( - e) ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. - The 'extern "C"' is for builds by C++ compilers; - although this is not generally supported in C code supporting it here - has little cost and some practical benefit (sr 110532). */ -#ifdef __cplusplus -extern "C" +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include #endif -char initscr (void); + int main (void) { -return initscr (); + + #ifndef immedok + void *x=immedok + #endif + ; return 0; } _ACEOF -for ac_lib in '' ncursesw ncurses -do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO" -then : - ac_cv_search_initscr=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext - if test ${ac_cv_search_initscr+y} -then : - break -fi -done -if test ${ac_cv_search_initscr+y} +if ac_fn_c_try_compile "$LINENO" then : - + ac_cv_lib_curses_immedok=yes else case e in #( - e) ac_cv_search_initscr=no ;; + e) ac_cv_lib_curses_immedok=no ;; esac fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS ;; +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_initscr" >&5 -printf "%s\n" "$ac_cv_search_initscr" >&6; } -ac_res=$ac_cv_search_initscr -if test "$ac_res" != no -then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - if test "x$have_curses" = xno +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_immedok" >&5 +printf "%s\n" "$ac_cv_lib_curses_immedok" >&6; } + if test "x$ac_cv_lib_curses_immedok" = xyes then : - have_curses=yes - CURSES_LIBS=${CURSES_LIBS-"$ac_cv_search_initscr"} -fi -else case e in #( - e) have_curses=no ;; -esac + +printf "%s\n" "#define HAVE_CURSES_IMMEDOK 1" >>confdefs.h + fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing update_panels" >&5 -printf %s "checking for library containing update_panels... " >&6; } -if test ${ac_cv_search_update_panels+y} + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function syncok" >&5 +printf %s "checking for curses function syncok... " >&6; } +if test ${ac_cv_lib_curses_syncok+y} then : printf %s "(cached) " >&6 else case e in #( - e) ac_func_search_save_LIBS=$LIBS -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. - The 'extern "C"' is for builds by C++ compilers; - although this is not generally supported in C code supporting it here - has little cost and some practical benefit (sr 110532). */ -#ifdef __cplusplus -extern "C" +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include #endif -char update_panels (void); + int main (void) { -return update_panels (); + + #ifndef syncok + void *x=syncok + #endif + ; return 0; } _ACEOF -for ac_lib in '' panelw panel -do - if test -z "$ac_lib"; then - ac_res="none required" - else - ac_res=-l$ac_lib - LIBS="-l$ac_lib $ac_func_search_save_LIBS" - fi - if ac_fn_c_try_link "$LINENO" -then : - ac_cv_search_update_panels=$ac_res -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext - if test ${ac_cv_search_update_panels+y} -then : - break -fi -done -if test ${ac_cv_search_update_panels+y} +if ac_fn_c_try_compile "$LINENO" then : - + ac_cv_lib_curses_syncok=yes else case e in #( - e) ac_cv_search_update_panels=no ;; + e) ac_cv_lib_curses_syncok=no ;; esac fi -rm conftest.$ac_ext -LIBS=$ac_func_search_save_LIBS ;; +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_update_panels" >&5 -printf "%s\n" "$ac_cv_search_update_panels" >&6; } -ac_res=$ac_cv_search_update_panels -if test "$ac_res" != no -then : - test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" - if test "x$have_panel" = xno +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_syncok" >&5 +printf "%s\n" "$ac_cv_lib_curses_syncok" >&6; } + if test "x$ac_cv_lib_curses_syncok" = xyes then : - have_panel=yes - PANEL_LIBS=${PANEL_LIBS-"$ac_cv_search_update_panels"} -fi -else case e in #( - e) have_panel=no ;; -esac -fi - - +printf "%s\n" "#define HAVE_CURSES_SYNCOK 1" >>confdefs.h -if test "have_curses" != "no" -then : - -CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') +fi -if test "x$ac_sys_system" = xDarwin -then : - as_fn_append CURSES_CFLAGS " -D_XOPEN_SOURCE_EXTENDED=1" -fi -PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wchgat" >&5 +printf %s "checking for curses function wchgat... " >&6; } +if test ${ac_cv_lib_curses_wchgat+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ -# On Solaris, term.h requires curses.h -ac_fn_c_check_header_compile "$LINENO" "term.h" "ac_cv_header_term_h" " #define NCURSES_OPAQUE 0 #if defined(HAVE_NCURSESW_NCURSES_H) # include @@ -29842,18 +31508,45 @@ ac_fn_c_check_header_compile "$LINENO" "term.h" "ac_cv_header_term_h" " # include #endif -" -if test "x$ac_cv_header_term_h" = xyes +int +main (void) +{ + + #ifndef wchgat + void *x=wchgat + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" then : - printf "%s\n" "#define HAVE_TERM_H 1" >>confdefs.h + ac_cv_lib_curses_wchgat=yes +else case e in #( + e) ac_cv_lib_curses_wchgat=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wchgat" >&5 +printf "%s\n" "$ac_cv_lib_curses_wchgat" >&6; } + if test "x$ac_cv_lib_curses_wchgat" = xyes +then : + +printf "%s\n" "#define HAVE_CURSES_WCHGAT 1" >>confdefs.h fi -# On HP/UX 11.0, mvwdelch is a block with a return statement -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether mvwdelch is an expression" >&5 -printf %s "checking whether mvwdelch is an expression... " >&6; } -if test ${ac_cv_mvwdelch_is_expression+y} + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function filter" >&5 +printf %s "checking for curses function filter... " >&6; } +if test ${ac_cv_lib_curses_filter+y} then : printf %s "(cached) " >&6 else case e in #( @@ -29879,8 +31572,9 @@ int main (void) { - int rtn; - rtn = mvwdelch(0,0,0); + #ifndef filter + void *x=filter + #endif ; return 0; @@ -29888,27 +31582,31 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_mvwdelch_is_expression=yes + ac_cv_lib_curses_filter=yes else case e in #( - e) ac_cv_mvwdelch_is_expression=no ;; + e) ac_cv_lib_curses_filter=no ;; esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_mvwdelch_is_expression" >&5 -printf "%s\n" "$ac_cv_mvwdelch_is_expression" >&6; } - -if test "$ac_cv_mvwdelch_is_expression" = yes -then +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_filter" >&5 +printf "%s\n" "$ac_cv_lib_curses_filter" >&6; } + if test "x$ac_cv_lib_curses_filter" = xyes +then : -printf "%s\n" "#define MVWDELCH_IS_EXPRESSION 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_FILTER 1" >>confdefs.h fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether WINDOW has _flags" >&5 -printf %s "checking whether WINDOW has _flags... " >&6; } -if test ${ac_cv_window_has_flags+y} + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function nofilter" >&5 +printf %s "checking for curses function nofilter... " >&6; } +if test ${ac_cv_lib_curses_nofilter+y} then : printf %s "(cached) " >&6 else case e in #( @@ -29934,8 +31632,9 @@ int main (void) { - WINDOW *w; - w->_flags = 0; + #ifndef nofilter + void *x=nofilter + #endif ; return 0; @@ -29943,22 +31642,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_window_has_flags=yes + ac_cv_lib_curses_nofilter=yes else case e in #( - e) ac_cv_window_has_flags=no ;; + e) ac_cv_lib_curses_nofilter=no ;; esac fi -rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_window_has_flags" >&5 -printf "%s\n" "$ac_cv_window_has_flags" >&6; } - - -if test "$ac_cv_window_has_flags" = yes -then +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_nofilter" >&5 +printf "%s\n" "$ac_cv_lib_curses_nofilter" >&6; } + if test "x$ac_cv_lib_curses_nofilter" = xyes +then : -printf "%s\n" "#define WINDOW_HAS_FLAGS 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_NOFILTER 1" >>confdefs.h fi @@ -29966,11 +31664,9 @@ fi - - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_pad" >&5 -printf %s "checking for curses function is_pad... " >&6; } -if test ${ac_cv_lib_curses_is_pad+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function has_key" >&5 +printf %s "checking for curses function has_key... " >&6; } +if test ${ac_cv_lib_curses_has_key+y} then : printf %s "(cached) " >&6 else case e in #( @@ -29996,8 +31692,8 @@ int main (void) { - #ifndef is_pad - void *x=is_pad + #ifndef has_key + void *x=has_key #endif ; @@ -30006,21 +31702,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_is_pad=yes + ac_cv_lib_curses_has_key=yes else case e in #( - e) ac_cv_lib_curses_is_pad=no ;; + e) ac_cv_lib_curses_has_key=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_pad" >&5 -printf "%s\n" "$ac_cv_lib_curses_is_pad" >&6; } - if test "x$ac_cv_lib_curses_is_pad" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_key" >&5 +printf "%s\n" "$ac_cv_lib_curses_has_key" >&6; } + if test "x$ac_cv_lib_curses_has_key" = xyes then : -printf "%s\n" "#define HAVE_CURSES_IS_PAD 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_HAS_KEY 1" >>confdefs.h fi @@ -30028,9 +31724,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_term_resized" >&5 -printf %s "checking for curses function is_term_resized... " >&6; } -if test ${ac_cv_lib_curses_is_term_resized+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function has_mouse" >&5 +printf %s "checking for curses function has_mouse... " >&6; } +if test ${ac_cv_lib_curses_has_mouse+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30056,8 +31752,8 @@ int main (void) { - #ifndef is_term_resized - void *x=is_term_resized + #ifndef has_mouse + void *x=has_mouse #endif ; @@ -30066,21 +31762,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_is_term_resized=yes + ac_cv_lib_curses_has_mouse=yes else case e in #( - e) ac_cv_lib_curses_is_term_resized=no ;; + e) ac_cv_lib_curses_has_mouse=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_term_resized" >&5 -printf "%s\n" "$ac_cv_lib_curses_is_term_resized" >&6; } - if test "x$ac_cv_lib_curses_is_term_resized" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_mouse" >&5 +printf "%s\n" "$ac_cv_lib_curses_has_mouse" >&6; } + if test "x$ac_cv_lib_curses_has_mouse" = xyes then : -printf "%s\n" "#define HAVE_CURSES_IS_TERM_RESIZED 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_HAS_MOUSE 1" >>confdefs.h fi @@ -30088,9 +31784,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function resize_term" >&5 -printf %s "checking for curses function resize_term... " >&6; } -if test ${ac_cv_lib_curses_resize_term+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_keypad" >&5 +printf %s "checking for curses function is_keypad... " >&6; } +if test ${ac_cv_lib_curses_is_keypad+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30116,8 +31812,8 @@ int main (void) { - #ifndef resize_term - void *x=resize_term + #ifndef is_keypad + void *x=is_keypad #endif ; @@ -30126,21 +31822,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_resize_term=yes + ac_cv_lib_curses_is_keypad=yes else case e in #( - e) ac_cv_lib_curses_resize_term=no ;; + e) ac_cv_lib_curses_is_keypad=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_resize_term" >&5 -printf "%s\n" "$ac_cv_lib_curses_resize_term" >&6; } - if test "x$ac_cv_lib_curses_resize_term" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_keypad" >&5 +printf "%s\n" "$ac_cv_lib_curses_is_keypad" >&6; } + if test "x$ac_cv_lib_curses_is_keypad" = xyes then : -printf "%s\n" "#define HAVE_CURSES_RESIZE_TERM 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_IS_KEYPAD 1" >>confdefs.h fi @@ -30148,9 +31844,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function resizeterm" >&5 -printf %s "checking for curses function resizeterm... " >&6; } -if test ${ac_cv_lib_curses_resizeterm+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_leaveok" >&5 +printf %s "checking for curses function is_leaveok... " >&6; } +if test ${ac_cv_lib_curses_is_leaveok+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30176,8 +31872,8 @@ int main (void) { - #ifndef resizeterm - void *x=resizeterm + #ifndef is_leaveok + void *x=is_leaveok #endif ; @@ -30186,21 +31882,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_resizeterm=yes + ac_cv_lib_curses_is_leaveok=yes else case e in #( - e) ac_cv_lib_curses_resizeterm=no ;; + e) ac_cv_lib_curses_is_leaveok=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_resizeterm" >&5 -printf "%s\n" "$ac_cv_lib_curses_resizeterm" >&6; } - if test "x$ac_cv_lib_curses_resizeterm" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_leaveok" >&5 +printf "%s\n" "$ac_cv_lib_curses_is_leaveok" >&6; } + if test "x$ac_cv_lib_curses_is_leaveok" = xyes then : -printf "%s\n" "#define HAVE_CURSES_RESIZETERM 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_IS_LEAVEOK 1" >>confdefs.h fi @@ -30208,9 +31904,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function immedok" >&5 -printf %s "checking for curses function immedok... " >&6; } -if test ${ac_cv_lib_curses_immedok+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function typeahead" >&5 +printf %s "checking for curses function typeahead... " >&6; } +if test ${ac_cv_lib_curses_typeahead+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30236,8 +31932,8 @@ int main (void) { - #ifndef immedok - void *x=immedok + #ifndef typeahead + void *x=typeahead #endif ; @@ -30246,21 +31942,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_immedok=yes + ac_cv_lib_curses_typeahead=yes else case e in #( - e) ac_cv_lib_curses_immedok=no ;; + e) ac_cv_lib_curses_typeahead=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_immedok" >&5 -printf "%s\n" "$ac_cv_lib_curses_immedok" >&6; } - if test "x$ac_cv_lib_curses_immedok" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_typeahead" >&5 +printf "%s\n" "$ac_cv_lib_curses_typeahead" >&6; } + if test "x$ac_cv_lib_curses_typeahead" = xyes then : -printf "%s\n" "#define HAVE_CURSES_IMMEDOK 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_TYPEAHEAD 1" >>confdefs.h fi @@ -30268,9 +31964,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function syncok" >&5 -printf %s "checking for curses function syncok... " >&6; } -if test ${ac_cv_lib_curses_syncok+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_env" >&5 +printf %s "checking for curses function use_env... " >&6; } +if test ${ac_cv_lib_curses_use_env+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30296,8 +31992,8 @@ int main (void) { - #ifndef syncok - void *x=syncok + #ifndef use_env + void *x=use_env #endif ; @@ -30306,21 +32002,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_syncok=yes + ac_cv_lib_curses_use_env=yes else case e in #( - e) ac_cv_lib_curses_syncok=no ;; + e) ac_cv_lib_curses_use_env=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_syncok" >&5 -printf "%s\n" "$ac_cv_lib_curses_syncok" >&6; } - if test "x$ac_cv_lib_curses_syncok" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_env" >&5 +printf "%s\n" "$ac_cv_lib_curses_use_env" >&6; } + if test "x$ac_cv_lib_curses_use_env" = xyes then : -printf "%s\n" "#define HAVE_CURSES_SYNCOK 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_USE_ENV 1" >>confdefs.h fi @@ -30328,9 +32024,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wchgat" >&5 -printf %s "checking for curses function wchgat... " >&6; } -if test ${ac_cv_lib_curses_wchgat+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function new_prescr" >&5 +printf %s "checking for curses function new_prescr... " >&6; } +if test ${ac_cv_lib_curses_new_prescr+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30356,8 +32052,8 @@ int main (void) { - #ifndef wchgat - void *x=wchgat + #ifndef new_prescr + void *x=new_prescr #endif ; @@ -30366,21 +32062,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_wchgat=yes + ac_cv_lib_curses_new_prescr=yes else case e in #( - e) ac_cv_lib_curses_wchgat=no ;; + e) ac_cv_lib_curses_new_prescr=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wchgat" >&5 -printf "%s\n" "$ac_cv_lib_curses_wchgat" >&6; } - if test "x$ac_cv_lib_curses_wchgat" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_new_prescr" >&5 +printf "%s\n" "$ac_cv_lib_curses_new_prescr" >&6; } + if test "x$ac_cv_lib_curses_new_prescr" = xyes then : -printf "%s\n" "#define HAVE_CURSES_WCHGAT 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_NEW_PRESCR 1" >>confdefs.h fi @@ -30388,9 +32084,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function filter" >&5 -printf %s "checking for curses function filter... " >&6; } -if test ${ac_cv_lib_curses_filter+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_screen" >&5 +printf %s "checking for curses function use_screen... " >&6; } +if test ${ac_cv_lib_curses_use_screen+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30416,8 +32112,8 @@ int main (void) { - #ifndef filter - void *x=filter + #ifndef use_screen + void *x=use_screen #endif ; @@ -30426,21 +32122,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_filter=yes + ac_cv_lib_curses_use_screen=yes else case e in #( - e) ac_cv_lib_curses_filter=no ;; + e) ac_cv_lib_curses_use_screen=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_filter" >&5 -printf "%s\n" "$ac_cv_lib_curses_filter" >&6; } - if test "x$ac_cv_lib_curses_filter" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_screen" >&5 +printf "%s\n" "$ac_cv_lib_curses_use_screen" >&6; } + if test "x$ac_cv_lib_curses_use_screen" = xyes then : -printf "%s\n" "#define HAVE_CURSES_FILTER 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_USE_SCREEN 1" >>confdefs.h fi @@ -30448,9 +32144,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function nofilter" >&5 -printf %s "checking for curses function nofilter... " >&6; } -if test ${ac_cv_lib_curses_nofilter+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_window" >&5 +printf %s "checking for curses function use_window... " >&6; } +if test ${ac_cv_lib_curses_use_window+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30476,8 +32172,8 @@ int main (void) { - #ifndef nofilter - void *x=nofilter + #ifndef use_window + void *x=use_window #endif ; @@ -30486,21 +32182,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_nofilter=yes + ac_cv_lib_curses_use_window=yes else case e in #( - e) ac_cv_lib_curses_nofilter=no ;; + e) ac_cv_lib_curses_use_window=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_nofilter" >&5 -printf "%s\n" "$ac_cv_lib_curses_nofilter" >&6; } - if test "x$ac_cv_lib_curses_nofilter" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_window" >&5 +printf "%s\n" "$ac_cv_lib_curses_use_window" >&6; } + if test "x$ac_cv_lib_curses_use_window" = xyes then : -printf "%s\n" "#define HAVE_CURSES_NOFILTER 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_USE_WINDOW 1" >>confdefs.h fi @@ -30508,9 +32204,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function has_key" >&5 -printf %s "checking for curses function has_key... " >&6; } -if test ${ac_cv_lib_curses_has_key+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function key_defined" >&5 +printf %s "checking for curses function key_defined... " >&6; } +if test ${ac_cv_lib_curses_key_defined+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30536,8 +32232,8 @@ int main (void) { - #ifndef has_key - void *x=has_key + #ifndef key_defined + void *x=key_defined #endif ; @@ -30546,31 +32242,31 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_has_key=yes + ac_cv_lib_curses_key_defined=yes else case e in #( - e) ac_cv_lib_curses_has_key=no ;; + e) ac_cv_lib_curses_key_defined=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_key" >&5 -printf "%s\n" "$ac_cv_lib_curses_has_key" >&6; } - if test "x$ac_cv_lib_curses_has_key" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_key_defined" >&5 +printf "%s\n" "$ac_cv_lib_curses_key_defined" >&6; } + if test "x$ac_cv_lib_curses_key_defined" = xyes then : -printf "%s\n" "#define HAVE_CURSES_HAS_KEY 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_KEY_DEFINED 1" >>confdefs.h fi - - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function has_mouse" >&5 -printf %s "checking for curses function has_mouse... " >&6; } -if test ${ac_cv_lib_curses_has_mouse+y} + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function term_attrs" >&5 +printf %s "checking for curses function term_attrs... " >&6; } +if test ${ac_cv_lib_curses_term_attrs+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30596,8 +32292,8 @@ int main (void) { - #ifndef has_mouse - void *x=has_mouse + #ifndef term_attrs + void *x=term_attrs #endif ; @@ -30606,21 +32302,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_has_mouse=yes + ac_cv_lib_curses_term_attrs=yes else case e in #( - e) ac_cv_lib_curses_has_mouse=no ;; + e) ac_cv_lib_curses_term_attrs=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_mouse" >&5 -printf "%s\n" "$ac_cv_lib_curses_has_mouse" >&6; } - if test "x$ac_cv_lib_curses_has_mouse" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_term_attrs" >&5 +printf "%s\n" "$ac_cv_lib_curses_term_attrs" >&6; } + if test "x$ac_cv_lib_curses_term_attrs" = xyes then : -printf "%s\n" "#define HAVE_CURSES_HAS_MOUSE 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_TERM_ATTRS 1" >>confdefs.h fi @@ -30628,9 +32324,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_keypad" >&5 -printf %s "checking for curses function is_keypad... " >&6; } -if test ${ac_cv_lib_curses_is_keypad+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function define_key" >&5 +printf %s "checking for curses function define_key... " >&6; } +if test ${ac_cv_lib_curses_define_key+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30656,8 +32352,8 @@ int main (void) { - #ifndef is_keypad - void *x=is_keypad + #ifndef define_key + void *x=define_key #endif ; @@ -30666,21 +32362,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_is_keypad=yes + ac_cv_lib_curses_define_key=yes else case e in #( - e) ac_cv_lib_curses_is_keypad=no ;; + e) ac_cv_lib_curses_define_key=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_keypad" >&5 -printf "%s\n" "$ac_cv_lib_curses_is_keypad" >&6; } - if test "x$ac_cv_lib_curses_is_keypad" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_define_key" >&5 +printf "%s\n" "$ac_cv_lib_curses_define_key" >&6; } + if test "x$ac_cv_lib_curses_define_key" = xyes then : -printf "%s\n" "#define HAVE_CURSES_IS_KEYPAD 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_DEFINE_KEY 1" >>confdefs.h fi @@ -30688,9 +32384,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_leaveok" >&5 -printf %s "checking for curses function is_leaveok... " >&6; } -if test ${ac_cv_lib_curses_is_leaveok+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function keyok" >&5 +printf %s "checking for curses function keyok... " >&6; } +if test ${ac_cv_lib_curses_keyok+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30716,8 +32412,8 @@ int main (void) { - #ifndef is_leaveok - void *x=is_leaveok + #ifndef keyok + void *x=keyok #endif ; @@ -30726,21 +32422,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_is_leaveok=yes + ac_cv_lib_curses_keyok=yes else case e in #( - e) ac_cv_lib_curses_is_leaveok=no ;; + e) ac_cv_lib_curses_keyok=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_is_leaveok" >&5 -printf "%s\n" "$ac_cv_lib_curses_is_leaveok" >&6; } - if test "x$ac_cv_lib_curses_is_leaveok" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_keyok" >&5 +printf "%s\n" "$ac_cv_lib_curses_keyok" >&6; } + if test "x$ac_cv_lib_curses_keyok" = xyes then : -printf "%s\n" "#define HAVE_CURSES_IS_LEAVEOK 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_KEYOK 1" >>confdefs.h fi @@ -30748,9 +32444,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function typeahead" >&5 -printf %s "checking for curses function typeahead... " >&6; } -if test ${ac_cv_lib_curses_typeahead+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function set_escdelay" >&5 +printf %s "checking for curses function set_escdelay... " >&6; } +if test ${ac_cv_lib_curses_set_escdelay+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30776,8 +32472,8 @@ int main (void) { - #ifndef typeahead - void *x=typeahead + #ifndef set_escdelay + void *x=set_escdelay #endif ; @@ -30786,21 +32482,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_typeahead=yes + ac_cv_lib_curses_set_escdelay=yes else case e in #( - e) ac_cv_lib_curses_typeahead=no ;; + e) ac_cv_lib_curses_set_escdelay=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_typeahead" >&5 -printf "%s\n" "$ac_cv_lib_curses_typeahead" >&6; } - if test "x$ac_cv_lib_curses_typeahead" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_set_escdelay" >&5 +printf "%s\n" "$ac_cv_lib_curses_set_escdelay" >&6; } + if test "x$ac_cv_lib_curses_set_escdelay" = xyes then : -printf "%s\n" "#define HAVE_CURSES_TYPEAHEAD 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_SET_ESCDELAY 1" >>confdefs.h fi @@ -30808,9 +32504,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_env" >&5 -printf %s "checking for curses function use_env... " >&6; } -if test ${ac_cv_lib_curses_use_env+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function set_tabsize" >&5 +printf %s "checking for curses function set_tabsize... " >&6; } +if test ${ac_cv_lib_curses_set_tabsize+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30836,8 +32532,8 @@ int main (void) { - #ifndef use_env - void *x=use_env + #ifndef set_tabsize + void *x=set_tabsize #endif ; @@ -30846,21 +32542,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_use_env=yes + ac_cv_lib_curses_set_tabsize=yes else case e in #( - e) ac_cv_lib_curses_use_env=no ;; + e) ac_cv_lib_curses_set_tabsize=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_env" >&5 -printf "%s\n" "$ac_cv_lib_curses_use_env" >&6; } - if test "x$ac_cv_lib_curses_use_env" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_set_tabsize" >&5 +printf "%s\n" "$ac_cv_lib_curses_set_tabsize" >&6; } + if test "x$ac_cv_lib_curses_set_tabsize" = xyes then : -printf "%s\n" "#define HAVE_CURSES_USE_ENV 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_SET_TABSIZE 1" >>confdefs.h fi @@ -30868,9 +32564,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function new_prescr" >&5 -printf %s "checking for curses function new_prescr... " >&6; } -if test ${ac_cv_lib_curses_new_prescr+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_get" >&5 +printf %s "checking for curses function wattr_get... " >&6; } +if test ${ac_cv_lib_curses_wattr_get+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30896,8 +32592,8 @@ int main (void) { - #ifndef new_prescr - void *x=new_prescr + #ifndef wattr_get + void *x=wattr_get #endif ; @@ -30906,21 +32602,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_new_prescr=yes + ac_cv_lib_curses_wattr_get=yes else case e in #( - e) ac_cv_lib_curses_new_prescr=no ;; + e) ac_cv_lib_curses_wattr_get=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_new_prescr" >&5 -printf "%s\n" "$ac_cv_lib_curses_new_prescr" >&6; } - if test "x$ac_cv_lib_curses_new_prescr" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_get" >&5 +printf "%s\n" "$ac_cv_lib_curses_wattr_get" >&6; } + if test "x$ac_cv_lib_curses_wattr_get" = xyes then : -printf "%s\n" "#define HAVE_CURSES_NEW_PRESCR 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_WATTR_GET 1" >>confdefs.h fi @@ -30928,9 +32624,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_screen" >&5 -printf %s "checking for curses function use_screen... " >&6; } -if test ${ac_cv_lib_curses_use_screen+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_set" >&5 +printf %s "checking for curses function wattr_set... " >&6; } +if test ${ac_cv_lib_curses_wattr_set+y} then : printf %s "(cached) " >&6 else case e in #( @@ -30956,8 +32652,8 @@ int main (void) { - #ifndef use_screen - void *x=use_screen + #ifndef wattr_set + void *x=wattr_set #endif ; @@ -30966,21 +32662,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_use_screen=yes + ac_cv_lib_curses_wattr_set=yes else case e in #( - e) ac_cv_lib_curses_use_screen=no ;; + e) ac_cv_lib_curses_wattr_set=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_screen" >&5 -printf "%s\n" "$ac_cv_lib_curses_use_screen" >&6; } - if test "x$ac_cv_lib_curses_use_screen" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_set" >&5 +printf "%s\n" "$ac_cv_lib_curses_wattr_set" >&6; } + if test "x$ac_cv_lib_curses_wattr_set" = xyes then : -printf "%s\n" "#define HAVE_CURSES_USE_SCREEN 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_WATTR_SET 1" >>confdefs.h fi @@ -30988,9 +32684,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_window" >&5 -printf %s "checking for curses function use_window... " >&6; } -if test ${ac_cv_lib_curses_use_window+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_on" >&5 +printf %s "checking for curses function wattr_on... " >&6; } +if test ${ac_cv_lib_curses_wattr_on+y} then : printf %s "(cached) " >&6 else case e in #( @@ -31016,8 +32712,8 @@ int main (void) { - #ifndef use_window - void *x=use_window + #ifndef wattr_on + void *x=wattr_on #endif ; @@ -31026,21 +32722,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_use_window=yes + ac_cv_lib_curses_wattr_on=yes else case e in #( - e) ac_cv_lib_curses_use_window=no ;; + e) ac_cv_lib_curses_wattr_on=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_window" >&5 -printf "%s\n" "$ac_cv_lib_curses_use_window" >&6; } - if test "x$ac_cv_lib_curses_use_window" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_on" >&5 +printf "%s\n" "$ac_cv_lib_curses_wattr_on" >&6; } + if test "x$ac_cv_lib_curses_wattr_on" = xyes then : -printf "%s\n" "#define HAVE_CURSES_USE_WINDOW 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_WATTR_ON 1" >>confdefs.h fi @@ -31048,9 +32744,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function key_defined" >&5 -printf %s "checking for curses function key_defined... " >&6; } -if test ${ac_cv_lib_curses_key_defined+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wattr_off" >&5 +printf %s "checking for curses function wattr_off... " >&6; } +if test ${ac_cv_lib_curses_wattr_off+y} then : printf %s "(cached) " >&6 else case e in #( @@ -31076,8 +32772,8 @@ int main (void) { - #ifndef key_defined - void *x=key_defined + #ifndef wattr_off + void *x=wattr_off #endif ; @@ -31086,21 +32782,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_key_defined=yes + ac_cv_lib_curses_wattr_off=yes else case e in #( - e) ac_cv_lib_curses_key_defined=no ;; + e) ac_cv_lib_curses_wattr_off=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_key_defined" >&5 -printf "%s\n" "$ac_cv_lib_curses_key_defined" >&6; } - if test "x$ac_cv_lib_curses_key_defined" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wattr_off" >&5 +printf "%s\n" "$ac_cv_lib_curses_wattr_off" >&6; } + if test "x$ac_cv_lib_curses_wattr_off" = xyes then : -printf "%s\n" "#define HAVE_CURSES_KEY_DEFINED 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_WATTR_OFF 1" >>confdefs.h fi @@ -31108,9 +32804,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function term_attrs" >&5 -printf %s "checking for curses function term_attrs... " >&6; } -if test ${ac_cv_lib_curses_term_attrs+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function wcolor_set" >&5 +printf %s "checking for curses function wcolor_set... " >&6; } +if test ${ac_cv_lib_curses_wcolor_set+y} then : printf %s "(cached) " >&6 else case e in #( @@ -31136,8 +32832,8 @@ int main (void) { - #ifndef term_attrs - void *x=term_attrs + #ifndef wcolor_set + void *x=wcolor_set #endif ; @@ -31146,21 +32842,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_term_attrs=yes + ac_cv_lib_curses_wcolor_set=yes else case e in #( - e) ac_cv_lib_curses_term_attrs=no ;; + e) ac_cv_lib_curses_wcolor_set=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_term_attrs" >&5 -printf "%s\n" "$ac_cv_lib_curses_term_attrs" >&6; } - if test "x$ac_cv_lib_curses_term_attrs" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wcolor_set" >&5 +printf "%s\n" "$ac_cv_lib_curses_wcolor_set" >&6; } + if test "x$ac_cv_lib_curses_wcolor_set" = xyes then : -printf "%s\n" "#define HAVE_CURSES_TERM_ATTRS 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_WCOLOR_SET 1" >>confdefs.h fi @@ -31168,9 +32864,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function define_key" >&5 -printf %s "checking for curses function define_key... " >&6; } -if test ${ac_cv_lib_curses_define_key+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_attr_on" >&5 +printf %s "checking for curses function slk_attr_on... " >&6; } +if test ${ac_cv_lib_curses_slk_attr_on+y} then : printf %s "(cached) " >&6 else case e in #( @@ -31196,8 +32892,8 @@ int main (void) { - #ifndef define_key - void *x=define_key + #ifndef slk_attr_on + void *x=slk_attr_on #endif ; @@ -31206,21 +32902,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_define_key=yes + ac_cv_lib_curses_slk_attr_on=yes else case e in #( - e) ac_cv_lib_curses_define_key=no ;; + e) ac_cv_lib_curses_slk_attr_on=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_define_key" >&5 -printf "%s\n" "$ac_cv_lib_curses_define_key" >&6; } - if test "x$ac_cv_lib_curses_define_key" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_attr_on" >&5 +printf "%s\n" "$ac_cv_lib_curses_slk_attr_on" >&6; } + if test "x$ac_cv_lib_curses_slk_attr_on" = xyes then : -printf "%s\n" "#define HAVE_CURSES_DEFINE_KEY 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_SLK_ATTR_ON 1" >>confdefs.h fi @@ -31228,9 +32924,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function keyok" >&5 -printf %s "checking for curses function keyok... " >&6; } -if test ${ac_cv_lib_curses_keyok+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_attr_off" >&5 +printf %s "checking for curses function slk_attr_off... " >&6; } +if test ${ac_cv_lib_curses_slk_attr_off+y} then : printf %s "(cached) " >&6 else case e in #( @@ -31256,8 +32952,8 @@ int main (void) { - #ifndef keyok - void *x=keyok + #ifndef slk_attr_off + void *x=slk_attr_off #endif ; @@ -31266,21 +32962,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_keyok=yes + ac_cv_lib_curses_slk_attr_off=yes else case e in #( - e) ac_cv_lib_curses_keyok=no ;; + e) ac_cv_lib_curses_slk_attr_off=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_keyok" >&5 -printf "%s\n" "$ac_cv_lib_curses_keyok" >&6; } - if test "x$ac_cv_lib_curses_keyok" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_attr_off" >&5 +printf "%s\n" "$ac_cv_lib_curses_slk_attr_off" >&6; } + if test "x$ac_cv_lib_curses_slk_attr_off" = xyes then : -printf "%s\n" "#define HAVE_CURSES_KEYOK 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_SLK_ATTR_OFF 1" >>confdefs.h fi @@ -31288,9 +32984,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function set_escdelay" >&5 -printf %s "checking for curses function set_escdelay... " >&6; } -if test ${ac_cv_lib_curses_set_escdelay+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_attr_set" >&5 +printf %s "checking for curses function slk_attr_set... " >&6; } +if test ${ac_cv_lib_curses_slk_attr_set+y} then : printf %s "(cached) " >&6 else case e in #( @@ -31316,8 +33012,8 @@ int main (void) { - #ifndef set_escdelay - void *x=set_escdelay + #ifndef slk_attr_set + void *x=slk_attr_set #endif ; @@ -31326,21 +33022,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_set_escdelay=yes + ac_cv_lib_curses_slk_attr_set=yes else case e in #( - e) ac_cv_lib_curses_set_escdelay=no ;; + e) ac_cv_lib_curses_slk_attr_set=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_set_escdelay" >&5 -printf "%s\n" "$ac_cv_lib_curses_set_escdelay" >&6; } - if test "x$ac_cv_lib_curses_set_escdelay" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_attr_set" >&5 +printf "%s\n" "$ac_cv_lib_curses_slk_attr_set" >&6; } + if test "x$ac_cv_lib_curses_slk_attr_set" = xyes then : -printf "%s\n" "#define HAVE_CURSES_SET_ESCDELAY 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_SLK_ATTR_SET 1" >>confdefs.h fi @@ -31348,9 +33044,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function set_tabsize" >&5 -printf %s "checking for curses function set_tabsize... " >&6; } -if test ${ac_cv_lib_curses_set_tabsize+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function slk_color" >&5 +printf %s "checking for curses function slk_color... " >&6; } +if test ${ac_cv_lib_curses_slk_color+y} then : printf %s "(cached) " >&6 else case e in #( @@ -31376,8 +33072,8 @@ int main (void) { - #ifndef set_tabsize - void *x=set_tabsize + #ifndef slk_color + void *x=slk_color #endif ; @@ -31386,21 +33082,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_set_tabsize=yes + ac_cv_lib_curses_slk_color=yes else case e in #( - e) ac_cv_lib_curses_set_tabsize=no ;; + e) ac_cv_lib_curses_slk_color=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_set_tabsize" >&5 -printf "%s\n" "$ac_cv_lib_curses_set_tabsize" >&6; } - if test "x$ac_cv_lib_curses_set_tabsize" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_slk_color" >&5 +printf "%s\n" "$ac_cv_lib_curses_slk_color" >&6; } + if test "x$ac_cv_lib_curses_slk_color" = xyes then : -printf "%s\n" "#define HAVE_CURSES_SET_TABSIZE 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_SLK_COLOR 1" >>confdefs.h fi @@ -31634,6 +33330,66 @@ fi + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function scr_set" >&5 +printf %s "checking for curses function scr_set... " >&6; } +if test ${ac_cv_lib_curses_scr_set+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include +#elif defined(HAVE_NCURSESW_CURSES_H) +# include +#elif defined(HAVE_NCURSES_NCURSES_H) +# include +#elif defined(HAVE_NCURSES_CURSES_H) +# include +#elif defined(HAVE_NCURSES_H) +# include +#elif defined(HAVE_CURSES_H) +# include +#endif + +int +main (void) +{ + + #ifndef scr_set + void *x=scr_set + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_scr_set=yes +else case e in #( + e) ac_cv_lib_curses_scr_set=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_scr_set" >&5 +printf "%s\n" "$ac_cv_lib_curses_scr_set" >&6; } + if test "x$ac_cv_lib_curses_scr_set" = xyes +then : + +printf "%s\n" "#define HAVE_CURSES_SCR_SET 1" >>confdefs.h + +fi + + + CPPFLAGS=$ac_save_cppflags fi diff --git a/configure.ac b/configure.ac index ad8280349466e06..93b211eef749880 100644 --- a/configure.ac +++ b/configure.ac @@ -935,6 +935,12 @@ then AC_DEFINE([_POSIX_C_SOURCE], [202405L], [Define to activate features from IEEE Std 1003.1-2024]) + + # Defining _POSIX_C_SOURCE and _XOPEN_SOURCE hides C23 library + # declarations on FreeBSD (e.g. sinpi() in math.h) when compiling + # with -std=c11. Defining _ISOC23_SOURCE makes them visible again. + AC_DEFINE([_ISOC23_SOURCE], [1], + [Define to activate ISO C23 library declarations]) fi # On HP-UX mbstate_t requires _INCLUDE__STDC_A1_SOURCE @@ -1090,6 +1096,8 @@ AC_CACHE_CHECK([for CC compiler name], [ac_cv_cc_name], [ cat > conftest.c <@], + [select the curses backend for the curses and _curses_panel + modules, or disable them (default: auto)])], + [AS_CASE([$with_curses], + [yes|auto], [with_curses=auto], + [ncursesw|ncurses|curses|no], [], + [AC_MSG_ERROR([proper usage is --with(out)-curses@<:@=ncursesw|ncurses|curses|no@:>@])])], + [with_curses=auto]) +AC_MSG_RESULT([$with_curses]) + dnl PY_CHECK_CURSES(LIBCURSES, LIBPANEL) dnl Sets 'have_curses' and 'have_panel'. dnl For the PKG_CHECK_MODULES() calls, we can safely reuse the first variable @@ -7108,34 +7157,54 @@ PKG_CHECK_MODULES([CURSES], [$1], PKG_CHECK_MODULES([PANEL], [$2], [AC_DEFINE([HAVE_]panel_var, [1], [Define if you have the '$2' library]) AS_VAR_SET([have_panel], [yes])], - [AS_VAR_SET([have_panel], [no])])], + [dnl pkgsrc renames the ncurses panel to "gnupanel" so it does not clash + dnl with a system libpanel; it Requires the same ncurses, so it matches. + PKG_CHECK_MODULES([PANEL], [gnupanel], + [AS_VAR_SET([have_panel], [yes])], + [AS_VAR_SET([have_panel], [no])])])], [AS_VAR_SET([have_curses], [no])]) AS_VAR_POPDEF([curses_var]) AS_VAR_POPDEF([panel_var])]) -# Check for ncursesw/panelw first. If that fails, try ncurses/panel. -PY_CHECK_CURSES([ncursesw], [panelw]) -AS_VAR_IF([have_curses], [no], - [PY_CHECK_CURSES([ncurses], [panel])]) +# Detect the selected backend. ncursesw/ncurses are found via pkg-config; +# native curses has no .pc file and is left to the header/link probes below. +# curses_libs/panel_libs drive the AC_SEARCH_LIBS fallback; for "no" they are +# empty so nothing links and have_curses stays "no". +AS_CASE([$with_curses], + [ncursesw], [PY_CHECK_CURSES([ncursesw], [panelw]) + curses_libs="ncursesw"; panel_libs="panelw gnupanel"], + [ncurses], [PY_CHECK_CURSES([ncurses], [panel]) + curses_libs="ncurses"; panel_libs="panel gnupanel"], + [curses], [curses_libs="curses"; panel_libs="panel"], + [no], [curses_libs=""; panel_libs=""], + [dnl auto: prefer ncursesw, fall back to ncurses; never native curses. + PY_CHECK_CURSES([ncursesw], [panelw]) + AS_VAR_IF([have_curses], [no], [PY_CHECK_CURSES([ncurses], [panel])]) + curses_libs="ncursesw ncurses"; panel_libs="panelw panel gnupanel"]) WITH_SAVE_ENV([ - # Make sure we've got the header defines. + # Make sure we've got the header defines. For the native "curses" backend, + # probe only the plain headers: a system may also have ncurses headers (e.g. + # ncurses/curses.h), and picking those while linking the native library mixes + # incompatible declarations (e.g. tparm()) with the native . AS_VAR_APPEND([CPPFLAGS], [" $CURSES_CFLAGS $PANEL_CFLAGS"]) - AC_CHECK_HEADERS(m4_normalize([ - ncursesw/curses.h ncursesw/ncurses.h ncursesw/panel.h - ncurses/curses.h ncurses/ncurses.h ncurses/panel.h - curses.h ncurses.h panel.h - ])) + AS_VAR_IF([with_curses], [curses], + [AC_CHECK_HEADERS([curses.h panel.h])], + [AC_CHECK_HEADERS(m4_normalize([ + ncursesw/curses.h ncursesw/ncurses.h ncursesw/panel.h + ncurses/curses.h ncurses/ncurses.h ncurses/panel.h + curses.h ncurses.h panel.h + ]))]) # Check that we're able to link with crucial curses/panel functions. This # also serves as a fallback in case pkg-config failed. AS_VAR_APPEND([LIBS], [" $CURSES_LIBS $PANEL_LIBS"]) - AC_SEARCH_LIBS([initscr], [ncursesw ncurses], + AC_SEARCH_LIBS([initscr], [$curses_libs], [AS_VAR_IF([have_curses], [no], [AS_VAR_SET([have_curses], [yes]) CURSES_LIBS=${CURSES_LIBS-"$ac_cv_search_initscr"}])], [AS_VAR_SET([have_curses], [no])]) - AC_SEARCH_LIBS([update_panels], [panelw panel], + AC_SEARCH_LIBS([update_panels], [$panel_libs], [AS_VAR_IF([have_panel], [no], [AS_VAR_SET([have_panel], [yes]) PANEL_LIBS=${PANEL_LIBS-"$ac_cv_search_update_panels"}])], @@ -7163,7 +7232,7 @@ AC_DEFUN([_CURSES_INCLUDES],dnl #endif ]) -AS_IF([test "have_curses" != "no"], [ +AS_IF([test "$have_curses" != "no"], [ dnl remove _XOPEN_SOURCE macro from curses cflags. pyconfig.h sets dnl the macro to 700. CURSES_CFLAGS=$(echo $CURSES_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') @@ -7183,6 +7252,34 @@ AS_VAR_IF([ac_sys_system], [Darwin], [ dnl pyconfig.h defines _XOPEN_SOURCE=700 PANEL_CFLAGS=$(echo $PANEL_CFLAGS | sed 's/-D_XOPEN_SOURCE=600//g') +dnl A curses that is not named "ncursesw" can still be wide-character capable: +dnl the system curses of NetBSD, or an ncurses built with --enable-widec that +dnl keeps the plain name (pkgsrc, macOS). Probe for the wide API and, if +dnl present, build the module wide by defining HAVE_NCURSESW. The "ncursesw" +dnl backend already defines it, so only the other backends are probed. +AS_CASE([$with_curses], [ncursesw|no], [], [ + dnl Use the adjusted CURSES_CFLAGS (e.g. macOS's -D_XOPEN_SOURCE_EXTENDED) + dnl so the wide-character declarations are visible to the probe. This runs + dnl inside an outer WITH_SAVE_ENV, whose single CPPFLAGS save slot is not + dnl reentrant, so save and restore CPPFLAGS with a dedicated variable here. + save_curses_cppflags=$CPPFLAGS + AS_VAR_APPEND([CPPFLAGS], [" $CURSES_CFLAGS"]) + AC_CACHE_CHECK([whether curses supports wide characters], + [ac_cv_curses_wide], + [AC_LINK_IFELSE( + [AC_LANG_PROGRAM([_CURSES_INCLUDES], [[ + cchar_t wcval; + setcchar(&wcval, L"x", A_NORMAL, 0, NULL); + add_wch(&wcval); + ]])], + [ac_cv_curses_wide=yes], + [ac_cv_curses_wide=no])]) + CPPFLAGS=$save_curses_cppflags + dnl HAVE_NCURSESW marks the wide (cchar_t) curses API; its template comes + dnl from the ncursesw pkg-config check above, so no description here. + AS_VAR_IF([ac_cv_curses_wide], [yes], [AC_DEFINE([HAVE_NCURSESW], [1])]) +]) + # On Solaris, term.h requires curses.h AC_CHECK_HEADERS([term.h], [], [], _CURSES_INCLUDES) @@ -7288,6 +7385,17 @@ PY_CHECK_CURSES_FUNC([define_key]) PY_CHECK_CURSES_FUNC([keyok]) PY_CHECK_CURSES_FUNC([set_escdelay]) PY_CHECK_CURSES_FUNC([set_tabsize]) +dnl The X/Open attr_t and soft-label attribute functions are absent on old +dnl SVr4 curses (e.g. illumos). +PY_CHECK_CURSES_FUNC([wattr_get]) +PY_CHECK_CURSES_FUNC([wattr_set]) +PY_CHECK_CURSES_FUNC([wattr_on]) +PY_CHECK_CURSES_FUNC([wattr_off]) +PY_CHECK_CURSES_FUNC([wcolor_set]) +PY_CHECK_CURSES_FUNC([slk_attr_on]) +PY_CHECK_CURSES_FUNC([slk_attr_off]) +PY_CHECK_CURSES_FUNC([slk_attr_set]) +PY_CHECK_CURSES_FUNC([slk_color]) PY_CHECK_CURSES_VAR([ESCDELAY]) PY_CHECK_CURSES_VAR([TABSIZE]) @@ -7303,9 +7411,10 @@ AS_VAR_IF([ac_cv_lib_curses_getmouse], [yes], [AC_DEFINE([HAVE_CURSES_GETMOUSE], [1], [Define if you have the 'getmouse' function with the X/Open signature.])]) -dnl scr_dump and its companions scr_restore/scr_init/scr_set are an inseparable -dnl group; probing scr_dump alone gates the whole family. +dnl scr_dump gates scr_dump/scr_restore/scr_init; scr_set is separate, since +dnl old SVr4 curses (e.g. illumos) has the former but not scr_set. PY_CHECK_CURSES_FUNC([scr_dump]) +PY_CHECK_CURSES_FUNC([scr_set]) CPPFLAGS=$ac_save_cppflags ])dnl have_curses != no ])dnl save env diff --git a/pyconfig.h.in b/pyconfig.h.in index 2f9b2140d9f191a..2658fe8116781db 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -257,12 +257,27 @@ /* Define if you have the 'scr_dump' function. */ #undef HAVE_CURSES_SCR_DUMP +/* Define if you have the 'scr_set' function. */ +#undef HAVE_CURSES_SCR_SET + /* Define if you have the 'set_escdelay' function. */ #undef HAVE_CURSES_SET_ESCDELAY /* Define if you have the 'set_tabsize' function. */ #undef HAVE_CURSES_SET_TABSIZE +/* Define if you have the 'slk_attr_off' function. */ +#undef HAVE_CURSES_SLK_ATTR_OFF + +/* Define if you have the 'slk_attr_on' function. */ +#undef HAVE_CURSES_SLK_ATTR_ON + +/* Define if you have the 'slk_attr_set' function. */ +#undef HAVE_CURSES_SLK_ATTR_SET + +/* Define if you have the 'slk_color' function. */ +#undef HAVE_CURSES_SLK_COLOR + /* Define if you have the 'syncok' function. */ #undef HAVE_CURSES_SYNCOK @@ -284,9 +299,24 @@ /* Define if you have the 'use_window' function. */ #undef HAVE_CURSES_USE_WINDOW +/* Define if you have the 'wattr_get' function. */ +#undef HAVE_CURSES_WATTR_GET + +/* Define if you have the 'wattr_off' function. */ +#undef HAVE_CURSES_WATTR_OFF + +/* Define if you have the 'wattr_on' function. */ +#undef HAVE_CURSES_WATTR_ON + +/* Define if you have the 'wattr_set' function. */ +#undef HAVE_CURSES_WATTR_SET + /* Define if you have the 'wchgat' function. */ #undef HAVE_CURSES_WCHGAT +/* Define if you have the 'wcolor_set' function. */ +#undef HAVE_CURSES_WCOLOR_SET + /* Define to 1 if you have the header file. */ #undef HAVE_DB_H @@ -1938,6 +1968,9 @@ /* The size of '_Bool', as computed by sizeof. */ #undef SIZEOF__BOOL +/* Platform tag, used in binary module extension filenames. */ +#undef SOABI_PLATFORM + /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS @@ -2113,6 +2146,9 @@ /* Define to include mbstate_t for mbrtowc */ #undef _INCLUDE__STDC_A1_SOURCE +/* Define to activate ISO C23 library declarations */ +#undef _ISOC23_SOURCE + /* This must be defined on some systems to enable large file support. */ #undef _LARGEFILE_SOURCE